You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/01/08 17:37:53 UTC
[couchdb] 01/01: Implement customizable chttpd statistics
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch add-chttpd-stats
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 82580d43c2783c583aac1ddb232ee4491ac686d7
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Nov 29 13:11:38 2018 -0600
Implement customizable chttpd statistics
---
src/chttpd/src/chttpd.erl | 2 +
src/chttpd/src/chttpd_db.erl | 81 ++++++++++++++++++++++++------
src/chttpd/src/chttpd_show.erl | 17 ++++++-
src/chttpd/src/chttpd_stats.erl | 107 ++++++++++++++++++++++++++++++++++++++++
src/chttpd/src/chttpd_view.erl | 17 ++++++-
5 files changed, 206 insertions(+), 18 deletions(-)
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 6558b1e..378674a 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -265,6 +265,7 @@ handle_request_int(MochiReq) ->
before_request(HttpReq) ->
try
+ chttpd_stats:init(),
chttpd_plugin:before_request(HttpReq)
catch Tag:Error ->
{error, catch_error(HttpReq, Tag, Error)}
@@ -280,6 +281,7 @@ after_request(HttpReq, HttpResp0) ->
{ok, HttpResp0#httpd_resp{status = aborted}}
end,
HttpResp2 = update_stats(HttpReq, HttpResp1),
+ chttpd_stats:report(HttpReq, HttpResp2),
maybe_log(HttpReq, HttpResp2),
HttpResp2.
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index b413e48..7f0b8da 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -131,6 +131,7 @@ changes_callback(start, #cacc{feed = continuous} = Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Acc#cacc.mochi, 200),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, Change}, #cacc{feed = continuous} = Acc) ->
+ chttpd_stats:incr_rows(),
Data = [?JSON_ENCODE(Change) | "\n"],
Len = iolist_size(Data),
maybe_flush_changes_feed(Acc, Data, Len);
@@ -154,6 +155,7 @@ changes_callback(start, #cacc{feed = eventsource} = Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, {ChangeProp}=Change}, #cacc{feed = eventsource} = Acc) ->
+ chttpd_stats:incr_rows(),
Seq = proplists:get_value(seq, ChangeProp),
Chunk = [
"data: ", ?JSON_ENCODE(Change),
@@ -185,6 +187,7 @@ changes_callback(start, Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [], FirstChunk),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, Change}, Acc) ->
+ chttpd_stats:incr_rows(),
Data = [Acc#cacc.prepend, ?JSON_ENCODE(Change)],
Len = iolist_size(Data),
maybe_flush_changes_feed(Acc, Data, Len);
@@ -392,8 +395,12 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
% async_batching
spawn(fun() ->
case catch(fabric:update_doc(Db, Doc2, Options)) of
- {ok, _} -> ok;
- {accepted, _} -> ok;
+ {ok, _} ->
+ chttpd_stats:incr_writes(),
+ ok;
+ {accepted, _} ->
+ chttpd_stats:incr_writes(),
+ ok;
Error ->
couch_log:debug("Batch doc error (~s): ~p",[DocId, Error])
end
@@ -409,8 +416,10 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
$/, couch_util:url_encode(DocId)]),
case fabric:update_doc(Db, Doc2, Options) of
{ok, NewRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 201;
{accepted, NewRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 202
end,
send_json(Req, HttpCode, [{"Location", DocUrl}], {[
@@ -483,11 +492,13 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
case fabric:update_docs(Db, Docs, Options2) of
{ok, Results} ->
% output the results
+ chttpd_stats:incr_writes(length(Results)),
DocResults = lists:zipwith(fun update_doc_result_to_json/2,
Docs, Results),
send_json(Req, 201, DocResults);
{accepted, Results} ->
% output the results
+ chttpd_stats:incr_writes(length(Results)),
DocResults = lists:zipwith(fun update_doc_result_to_json/2,
Docs, Results),
send_json(Req, 202, DocResults);
@@ -499,9 +510,11 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
false ->
case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
{ok, Errors} ->
+ chttpd_stats:incr_writes(length(Docs)),
ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
send_json(Req, 201, ErrorsJson);
{accepted, Errors} ->
+ chttpd_stats:incr_writes(length(Docs)),
ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
send_json(Req, 202, ErrorsJson)
end;
@@ -610,8 +623,12 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) ->
end,
couch_stats:increment_counter([couchdb, document_purges, total], length(IdsRevs2)),
Results2 = case fabric:purge_docs(Db, IdsRevs2, Options) of
- {ok, Results} -> Results;
- {accepted, Results} -> Results
+ {ok, Results} ->
+ chttpd_stats:incr_writes(length(Results)),
+ Results;
+ {accepted, Results} ->
+ chttpd_stats:incr_writes(length(Results)),
+ Results
end,
{Code, Json} = purge_results_to_json(IdsRevs2, Results2),
send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]});
@@ -797,7 +814,7 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
VAcc1 = VAcc0#vacc{resp=Resp0},
VAcc2 = lists:foldl(fun(Args, Acc0) ->
{ok, Acc1} = fabric:all_docs(Db, Options,
- fun couch_mrview_http:view_cb/2, Acc0, Args),
+ fun view_cb/2, Acc0, Args),
Acc1
end, VAcc1, ArgQueries),
{ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
@@ -811,9 +828,20 @@ all_docs_view(Req, Db, Keys, OP) ->
Options = [{user_ctx, Req#httpd.user_ctx}],
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
- {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args3),
+ {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args3),
{ok, Resp#vacc.resp}.
+view_cb({row, Row} = Msg, Acc) ->
+ case lists:keymember(doc, 1, Row) of
+ true -> chttpd_stats:incr_reads();
+ false -> ok
+ end,
+ chttpd_stats:incr_rows(),
+ couch_mrview_http:view_cb(Msg, Acc);
+
+view_cb(Msg, Acc) ->
+ couch_mrview_http:view_cb(Msg, Acc).
+
db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) ->
% check for the existence of the doc to handle the 404 case.
couch_doc_open(Db, DocId, nil, []),
@@ -848,6 +876,7 @@ db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) ->
{ok, []} when Revs == all ->
chttpd:send_error(Req, {not_found, missing});
{ok, Results} ->
+ chttpd_stats:incr_reads(length(Results)),
case MochiReq:accepts_content_type("multipart/mixed") of
false ->
{ok, Resp} = start_json_response(Req, 200),
@@ -895,8 +924,11 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
false ->
Rev = couch_doc:parse_rev(list_to_binary(couch_util:get_value("_rev", Form))),
Doc = case fabric:open_revs(Db, DocId, [Rev], []) of
- {ok, [{ok, Doc0}]} -> Doc0;
- {error, Error} -> throw(Error)
+ {ok, [{ok, Doc0}]} ->
+ chttpd_stats:incr_reads(),
+ Doc0;
+ {error, Error} ->
+ throw(Error)
end
end,
UpdatedAtts = [
@@ -922,8 +954,10 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
},
case fabric:update_doc(Db, NewDoc, Options) of
{ok, NewRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 201;
{accepted, NewRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 202
end,
send_json(Req, HttpCode, [{"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewRev)) ++ "\""}], {[
@@ -969,8 +1003,12 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
spawn(fun() ->
case catch(fabric:update_doc(Db, Doc, Options)) of
- {ok, _} -> ok;
- {accepted, _} -> ok;
+ {ok, _} ->
+ chttpd_stats:incr_writes(),
+ ok;
+ {accepted, _} ->
+ chttpd_stats:incr_writes(),
+ ok;
Error ->
couch_log:notice("Batch doc error (~s): ~p",[DocId, Error])
end
@@ -1001,8 +1039,10 @@ db_doc_req(#httpd{method='COPY', user_ctx=Ctx}=Req, Db, SourceDocId) ->
case fabric:update_doc(Db,
Doc#doc{id=TargetDocId, revs=TargetRevs}, [{user_ctx,Ctx}]) of
{ok, NewTargetRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 201;
{accepted, NewTargetRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 202
end,
% respond
@@ -1305,6 +1345,7 @@ couch_doc_open(Db, DocId, Rev, Options0) ->
nil -> % open most recent rev
case fabric:open_doc(Db, DocId, Options) of
{ok, Doc} ->
+ chttpd_stats:incr_reads(),
Doc;
Error ->
throw(Error)
@@ -1312,6 +1353,7 @@ couch_doc_open(Db, DocId, Rev, Options0) ->
_ -> % open a specific rev (deletions come back as stubs)
case fabric:open_revs(Db, DocId, [Rev], Options) of
{ok, [{ok, Doc}]} ->
+ chttpd_stats:incr_reads(),
Doc;
{ok, [{{not_found, missing}, Rev}]} ->
throw(not_found);
@@ -1486,9 +1528,13 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
#doc{id=DocId};
Rev ->
case fabric:open_revs(Db, DocId, [Rev], [{user_ctx,Ctx}]) of
- {ok, [{ok, Doc0}]} -> Doc0;
- {ok, [Error]} -> throw(Error);
- {error, Error} -> throw(Error)
+ {ok, [{ok, Doc0}]} ->
+ chttpd_stats:incr_reads(),
+ Doc0;
+ {ok, [Error]} ->
+ throw(Error);
+ {error, Error} ->
+ throw(Error)
end
end,
@@ -1499,8 +1545,10 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
case fabric:update_doc(Db, DocEdited, [{user_ctx,Ctx}, {w,W}]) of
{ok, UpdatedRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 201;
{accepted, UpdatedRev} ->
+ chttpd_stats:incr_writes(),
HttpCode = 202
end,
erlang:put(mochiweb_request_recv, true),
@@ -1886,8 +1934,11 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) ->
RevStr = couch_util:get_value(<<"rev">>, Props),
Error = {RevStr, <<"not_found">>, <<"missing">>},
{DocId, {error, Error}, Options};
- Results ->
- {DocId, Results, Options}
+ {ok, Resps} = Results ->
+ chttpd_stats:incr_reads(length(Resps)),
+ {DocId, Results, Options};
+ Else ->
+ {DocId, Else, Options}
end.
diff --git a/src/chttpd/src/chttpd_show.erl b/src/chttpd/src/chttpd_show.erl
index a724189..c3bf119 100644
--- a/src/chttpd/src/chttpd_show.erl
+++ b/src/chttpd/src/chttpd_show.erl
@@ -25,6 +25,7 @@
maybe_open_doc(Db, DocId, Options) ->
case fabric:open_doc(Db, DocId, Options) of
{ok, Doc} ->
+ chttpd_stats:incr_reads(),
Doc;
{not_found, _} ->
nil
@@ -135,6 +136,7 @@ send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) ->
NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}),
couch_doc:validate_docid(NewDoc#doc.id),
{UpdateResult, NewRev} = fabric:update_doc(Db, NewDoc, Options),
+ chttpd_stats:incr_writes(),
NewRevStr = couch_doc:rev_to_str(NewRev),
case {UpdateResult, NewRev} of
{ok, _} ->
@@ -201,7 +203,7 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
couch_util:get_nested_json_value(DDoc#doc.body, [<<"lists">>, LName]),
DbName = couch_db:name(Db),
{ok, VDoc} = ddoc_cache:open(DbName, <<"_design/", ViewDesignName/binary>>),
- CB = fun couch_mrview_show:list_cb/2,
+ CB = fun list_cb/2,
QueryArgs = couch_mrview_http:parse_params(Req, Keys),
Options = [{user_ctx, Req#httpd.user_ctx}],
couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) ->
@@ -220,6 +222,19 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
end
end).
+
+list_cb({row, Row} = Msg, Acc) ->
+ case lists:keymember(doc, 1, Row) of
+ true -> chttpd_stats:incr_reads();
+ false -> ok
+ end,
+ chttpd_stats:incr_rows(),
+ couch_mrview_show:list_cb(Msg, Acc);
+
+list_cb(Msg, Acc) ->
+ couch_mrview_show:list_cb(Msg, Acc).
+
+
% Maybe this is in the proplists API
% todo move to couch_util
json_apply_field(H, {L}) ->
diff --git a/src/chttpd/src/chttpd_stats.erl b/src/chttpd/src/chttpd_stats.erl
new file mode 100644
index 0000000..59ec926
--- /dev/null
+++ b/src/chttpd/src/chttpd_stats.erl
@@ -0,0 +1,107 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_stats).
+
+
+-export([
+ init/0,
+ report/2,
+
+ incr_reads/0,
+ incr_reads/1,
+
+ incr_writes/0,
+ incr_writes/1,
+
+ incr_rows/0,
+ incr_rows/1
+]).
+
+
+-record(st, {
+ reads = 0,
+ writes = 0,
+ rows = 0
+}).
+
+
+-define(KEY, chttpd_stats).
+
+
+init() ->
+ put(?KEY, #st{}).
+
+
+report(HttpReq, HttpResp) ->
+ try
+ case get(?KEY) of
+ #st{} = St ->
+ report(HttpReq, HttpResp, St);
+ _ ->
+ ok
+ end
+ catch T:R ->
+ S = erlang:get_stacktrace(),
+ Fmt = "Failed to report chttpd request stats: ~p:~p ~p",
+ couch_log:error(Fmt, [T, R, S])
+ end.
+
+
+report(HttpReq, HttpResp, St) ->
+ case config:get("chttpd", "stats_reporter") of
+ undefined ->
+ ok;
+ ModStr ->
+ Mod = list_to_existing_atom(ModStr),
+ #st{
+ reads = Reads,
+ writes = Writes,
+ rows = Rows
+ } = St,
+ Mod:report(HttpReq, HttpResp, Reads, Writes, Rows)
+ end.
+
+
+incr_reads() ->
+ incr(#st.reads, 1).
+
+
+incr_reads(N) when is_integer(N), N >= 0 ->
+ incr(#st.reads, N).
+
+
+incr_writes() ->
+ incr(#st.writes, 1).
+
+
+incr_writes(N) when is_integer(N), N >= 0 ->
+ incr(#st.writes, N).
+
+
+incr_rows() ->
+ incr(#st.rows, 1).
+
+
+incr_rows(N) when is_integer(N), N >= 0 ->
+ incr(#st.rows, N).
+
+
+incr(Idx, Count) ->
+ case get(?KEY) of
+ #st{} = St ->
+ Total = element(Idx, St) + Count,
+ NewSt = setelement(Idx, St, Total),
+ put(?KEY, NewSt);
+ _ ->
+ ok
+ end.
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index 1fce165..26107d7 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -33,7 +33,7 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
VAcc1 = VAcc0#vacc{resp=Resp0},
VAcc2 = lists:foldl(fun(Args, Acc0) ->
{ok, Acc1} = fabric:query_view(Db, Options, DDoc, ViewName,
- fun couch_mrview_http:view_cb/2, Acc0, Args),
+ fun view_cb/2, Acc0, Args),
Acc1
end, VAcc1, ArgQueries),
{ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
@@ -46,9 +46,22 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
VAcc = #vacc{db=Db, req=Req, threshold=Max},
Options = [{user_ctx, Req#httpd.user_ctx}],
{ok, Resp} = fabric:query_view(Db, Options, DDoc, ViewName,
- fun couch_mrview_http:view_cb/2, VAcc, Args),
+ fun view_cb/2, VAcc, Args),
{ok, Resp#vacc.resp}.
+
+view_cb({row, Row} = Msg, Acc) ->
+ case lists:keymember(doc, 1, Row) of
+ true -> chttpd_stats:incr_reads();
+ false -> ok
+ end,
+ chttpd_stats:incr_rows(),
+ couch_mrview_http:view_cb(Msg, Acc);
+
+view_cb(Msg, Acc) ->
+ couch_mrview_http:view_cb(Msg, Acc).
+
+
handle_view_req(#httpd{method='POST',
path_parts=[_, _, _, _, ViewName, <<"queries">>]}=Req, Db, DDoc) ->
chttpd:validate_ctype(Req, "application/json"),