You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2014/08/28 14:12:14 UTC
[38/50] couch commit: updated refs/heads/master to 9d0ac7d
Use the new couch_stats application
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/62004cd9
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/62004cd9
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/62004cd9
Branch: refs/heads/master
Commit: 62004cd9507ca2b19b6cf650cf4fc5843801457f
Parents: 0355465
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu Aug 21 01:05:11 2014 -0500
Committer: Robert Newson <rn...@apache.org>
Committed: Thu Aug 28 13:00:02 2014 +0100
----------------------------------------------------------------------
priv/stat_descriptions.cfg | 194 ++++++++++++++++----
src/couch.app.src | 24 ++-
src/couch_auth_cache.erl | 6 +-
src/couch_db.erl | 29 ++-
src/couch_db_updater.erl | 11 ++
src/couch_file.erl | 4 +-
src/couch_httpd.erl | 14 +-
src/couch_httpd_cors.erl | 2 +-
src/couch_httpd_db.erl | 14 +-
src/couch_httpd_stats_handlers.erl | 75 ++++----
src/couch_lru.erl | 1 +
src/couch_query_servers.erl | 1 +
src/couch_server.erl | 6 +-
src/couch_stats_aggregator.erl | 312 --------------------------------
src/couch_stats_collector.erl | 133 --------------
15 files changed, 268 insertions(+), 558 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/priv/stat_descriptions.cfg
----------------------------------------------------------------------
diff --git a/priv/stat_descriptions.cfg b/priv/stat_descriptions.cfg
index b80d768..d6e1586 100644
--- a/priv/stat_descriptions.cfg
+++ b/priv/stat_descriptions.cfg
@@ -14,37 +14,163 @@
% a trailing full-stop / period
% Please keep this in alphabetical order
-{couchdb, database_writes, "number of times a database was changed"}.
-{couchdb, database_reads, "number of times a document was read from a database"}.
-{couchdb, open_databases, "number of open databases"}.
-{couchdb, open_os_files, "number of file descriptors CouchDB has open"}.
-{couchdb, request_time, "length of a request inside CouchDB without MochiWeb"}.
-{couchdb, auth_cache_hits, "number of authentication cache hits"}.
-{couchdb, auth_cache_misses, "number of authentication cache misses"}.
-
-{httpd, bulk_requests, "number of bulk requests"}.
-{httpd, requests, "number of HTTP requests"}.
-{httpd, temporary_view_reads, "number of temporary view reads"}.
-{httpd, view_reads, "number of view reads"}.
-{httpd, clients_requesting_changes, "number of clients for continuous _changes"}.
-
-{httpd_request_methods, 'COPY', "number of HTTP COPY requests"}.
-{httpd_request_methods, 'DELETE', "number of HTTP DELETE requests"}.
-{httpd_request_methods, 'GET', "number of HTTP GET requests"}.
-{httpd_request_methods, 'HEAD', "number of HTTP HEAD requests"}.
-{httpd_request_methods, 'POST', "number of HTTP POST requests"}.
-{httpd_request_methods, 'PUT', "number of HTTP PUT requests"}.
-
-{httpd_status_codes, '200', "number of HTTP 200 OK responses"}.
-{httpd_status_codes, '201', "number of HTTP 201 Created responses"}.
-{httpd_status_codes, '202', "number of HTTP 202 Accepted responses"}.
-{httpd_status_codes, '301', "number of HTTP 301 Moved Permanently responses"}.
-{httpd_status_codes, '304', "number of HTTP 304 Not Modified responses"}.
-{httpd_status_codes, '400', "number of HTTP 400 Bad Request responses"}.
-{httpd_status_codes, '401', "number of HTTP 401 Unauthorized responses"}.
-{httpd_status_codes, '403', "number of HTTP 403 Forbidden responses"}.
-{httpd_status_codes, '404', "number of HTTP 404 Not Found responses"}.
-{httpd_status_codes, '405', "number of HTTP 405 Method Not Allowed responses"}.
-{httpd_status_codes, '409', "number of HTTP 409 Conflict responses"}.
-{httpd_status_codes, '412', "number of HTTP 412 Precondition Failed responses"}.
-{httpd_status_codes, '500', "number of HTTP 500 Internal Server Error responses"}.
+{[couchdb, auth_cache_hits], [
+ {type, counter},
+ {desc, <<"number of authentication cache hits">>}
+]}.
+{[couchdb, auth_cache_misses], [
+ {type, counter},
+ {desc, <<"number of authentication cache misses">>}
+]}.
+{[couchdb, collect_results_time], [
+ {type, histogram},
+ {desc, <<"microsecond latency for calls to couch_db:collect_results/3">>}
+]}.
+{[couchdb, database_writes], [
+ {type, counter},
+ {desc, <<"number of times a database was changed">>}
+]}.
+{[couchdb, database_reads], [
+ {type, counter},
+ {desc, <<"number of times a document was read from a database">>}
+]}.
+{[couchdb, db_open_time], [
+ {type, histogram},
+ {desc, <<"milliseconds required to open a database">>}
+]}.
+{[couchdb, document_inserts], [
+ {type, counter},
+ {desc, <<"number of documents inserted">>}
+]}.
+{[couchdb, document_writes], [
+ {type, counter},
+ {desc, <<"number of document write operations">>}
+]}.
+{[couchdb, local_document_writes], [
+ {type, counter},
+ {desc, <<"number of _local document write operations">>}
+]}.
+{[couchdb, httpd, bulk_requests], [
+ {type, counter},
+ {desc, <<"number of bulk requests">>}
+]}.
+{[couchdb, httpd, requests], [
+ {type, counter},
+ {desc, <<"number of HTTP requests">>}
+]}.
+{[couchdb, httpd, temporary_view_reads], [
+ {type, counter},
+ {desc, <<"number of temporary view reads">>}
+]}.
+{[couchdb, httpd, view_reads], [
+ {type, counter},
+ {desc, <<"number of view reads">>}
+]}.
+{[couchdb, httpd, clients_requesting_changes], [
+ {type, counter},
+ {desc, <<"number of clients for continuous _changes">>}
+]}.
+{[couchdb, httpd_request_methods, 'COPY'], [
+ {type, counter},
+ {desc, <<"number of HTTP COPY requests">>}
+]}.
+{[couchdb, httpd_request_methods, 'DELETE'], [
+ {type, counter},
+ {desc, <<"number of HTTP DELETE requests">>}
+]}.
+{[couchdb, httpd_request_methods, 'GET'], [
+ {type, counter},
+ {desc, <<"number of HTTP GET requests">>}
+]}.
+{[couchdb, httpd_request_methods, 'HEAD'], [
+ {type, counter},
+ {desc, <<"number of HTTP HEAD requests">>}
+]}.
+{[couchdb, httpd_request_methods, 'POST'], [
+ {type, counter},
+ {desc, <<"number of HTTP POST requests">>}
+]}.
+{[couchdb, httpd_request_methods, 'PUT'], [
+ {type, counter},
+ {desc, <<"number of HTTP PUT requests">>}
+]}.
+{[couchdb, httpd_status_codes, '200'], [
+ {type, counter},
+ {desc, <<"number of HTTP 200 OK responses">>}
+]}.
+{[couchdb, httpd_status_codes, '201'], [
+ {type, counter},
+ {desc, <<"number of HTTP 201 Created responses">>}
+]}.
+{[couchdb, httpd_status_codes, '202'], [
+ {type, counter},
+ {desc, <<"number of HTTP 202 Accepted responses">>}
+]}.
+{[couchdb, httpd_status_codes, '301'], [
+ {type, counter},
+ {desc, <<"number of HTTP 301 Moved Permanently responses">>}
+]}.
+{[couchdb, httpd_status_codes, '302'], [
+ {type, counter},
+ {desc, <<"number of HTTP 302 Found responses">>}
+]}.
+{[couchdb, httpd_status_codes, '304'], [
+ {type, counter},
+ {desc, <<"number of HTTP 304 Not Modified responses">>}
+]}.
+{[couchdb, httpd_status_codes, '400'], [
+ {type, counter},
+ {desc, <<"number of HTTP 400 Bad Request responses">>}
+]}.
+{[couchdb, httpd_status_codes, '401'], [
+ {type, counter},
+ {desc, <<"number of HTTP 401 Unauthorized responses">>}
+]}.
+{[couchdb, httpd_status_codes, '403'], [
+ {type, counter},
+ {desc, <<"number of HTTP 403 Forbidden responses">>}
+]}.
+{[couchdb, httpd_status_codes, '404'], [
+ {type, counter},
+ {desc, <<"number of HTTP 404 Not Found responses">>}
+]}.
+{[couchdb, httpd_status_codes, '405'], [
+ {type, counter},
+ {desc, <<"number of HTTP 405 Method Not Allowed responses">>}
+]}.
+{[couchdb, httpd_status_codes, '409'], [
+ {type, counter},
+ {desc, <<"number of HTTP 409 Conflict responses">>}
+]}.
+{[couchdb, httpd_status_codes, '412'], [
+ {type, counter},
+ {desc, <<"number of HTTP 412 Precondition Failed responses">>}
+]}.
+{[couchdb, httpd_status_codes, '500'], [
+ {type, counter},
+ {desc, <<"number of HTTP 500 Internal Server Error responses">>}
+]}.
+{[couchdb, open_databases], [
+ {type, counter},
+ {desc, <<"number of open databases">>}
+]}.
+{[couchdb, open_os_files], [
+ {type, counter},
+ {desc, <<"number of file descriptors CouchDB has open">>}
+]}.
+{[couchdb, request_time], [
+ {type, histogram},
+ {desc, <<"length of a request inside CouchDB without MochiWeb">>}
+]}.
+{[couchdb, couch_server, lru_skip], [
+ {type, counter},
+ {desc, <<"number of couch_server LRU operations skipped">>}
+]}.
+{[couchdb, couchjs, map_doc], [
+ {type, counter},
+ {desc, <<"number of documents mapped in the couchjs view server">>}
+]}.
+{[couchdb, couchjs, emits], [
+ {type, counter},
+ {desc, <<"number of invocations of `emit' in map functions in the couchjs view server">>}
+]}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch.app.src
----------------------------------------------------------------------
diff --git a/src/couch.app.src b/src/couch.app.src
index 30f3b2a..ba12f41 100644
--- a/src/couch.app.src
+++ b/src/couch.app.src
@@ -23,11 +23,27 @@
couch_secondary_services,
couch_server,
couch_sup,
- couch_stats_aggregator,
- couch_stats_collector,
couch_task_status
]},
{mod, {couch_app, []}},
- {applications, [kernel, stdlib, crypto, sasl, inets, oauth, ibrowse,
- mochiweb, ssl, couch_log, couch_event, b64url]}
+ {applications, [
+ % stdlib
+ kernel,
+ stdlib,
+ crypto,
+ sasl,
+ inets,
+ ssl,
+
+ % Upstream deps
+ ibrowse,
+ mochiweb,
+ oauth,
+
+ % ASF deps
+ b64url,
+ couch_log,
+ couch_event,
+ couch_stats
+ ]}
]}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_auth_cache.erl
----------------------------------------------------------------------
diff --git a/src/couch_auth_cache.erl b/src/couch_auth_cache.erl
index d53cff0..8cf631b 100644
--- a/src/couch_auth_cache.erl
+++ b/src/couch_auth_cache.erl
@@ -102,7 +102,7 @@ get_from_cache(UserName) ->
[] ->
gen_server:call(?MODULE, {fetch, UserName}, infinity);
[{UserName, {Credentials, _ATime}}] ->
- couch_stats_collector:increment({couchdb, auth_cache_hits}),
+ couch_stats:increment_counter([couchdb, auth_cache_hits]),
gen_server:cast(?MODULE, {cache_hit, UserName}),
Credentials
end
@@ -182,11 +182,11 @@ handle_call({new_max_cache_size, NewSize}, _From, State) ->
handle_call({fetch, UserName}, _From, State) ->
{Credentials, NewState} = case ets:lookup(?BY_USER, UserName) of
[{UserName, {Creds, ATime}}] ->
- couch_stats_collector:increment({couchdb, auth_cache_hits}),
+ couch_stats:increment_counter([couchdb, auth_cache_hits]),
cache_hit(UserName, Creds, ATime),
{Creds, State};
[] ->
- couch_stats_collector:increment({couchdb, auth_cache_misses}),
+ couch_stats:increment_counter([couchdb, auth_cache_misses]),
Creds = get_user_props_from_db(UserName),
State1 = add_cache_entry(UserName, Creds, erlang:now(), State),
{Creds, State1}
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_db.erl b/src/couch_db.erl
index 8a9f9e4..bd250ef 100644
--- a/src/couch_db.erl
+++ b/src/couch_db.erl
@@ -125,7 +125,8 @@ is_idle(#db{compactor_pid=nil, waiting_delayed_commit=nil} = Db) ->
undefined ->
true;
{monitored_by, Pids} ->
- (Pids -- [Db#db.main_pid, whereis(couch_stats_collector)]) =:= []
+ PidTracker = whereis(couch_stats_process_tracker),
+ (Pids -- [Db#db.main_pid, PidTracker]) =:= []
end;
is_idle(_Db) ->
false.
@@ -172,7 +173,7 @@ open_doc(Db, IdOrDocInfo) ->
open_doc(Db, IdOrDocInfo, []).
open_doc(Db, Id, Options) ->
- increment_stat(Db, {couchdb, database_reads}),
+ increment_stat(Db, [couchdb, database_reads]),
case open_doc_int(Db, Id, Options) of
{ok, #doc{deleted=true}=Doc} ->
case lists:member(deleted, Options) of
@@ -221,7 +222,7 @@ find_ancestor_rev_pos({RevPos, [RevId|Rest]}, AttsSinceRevs) ->
end.
open_doc_revs(Db, Id, Revs, Options) ->
- increment_stat(Db, {couchdb, database_reads}),
+ increment_stat(Db, [couchdb, database_reads]),
[{ok, Results}] = open_doc_revs_int(Db, [{Id, Revs}], Options),
{ok, [apply_open_options(Result, Options) || Result <- Results]}.
@@ -840,7 +841,7 @@ doc_tag(#doc{meta=Meta}) ->
end.
update_docs(Db, Docs0, Options, replicated_changes) ->
- increment_stat(Db, {couchdb, database_writes}),
+ increment_stat(Db, [couchdb, database_writes]),
Docs = tag_docs(Docs0),
DocBuckets = before_docs_update(Db, group_alike_docs(Docs)),
@@ -867,7 +868,7 @@ update_docs(Db, Docs0, Options, replicated_changes) ->
{ok, DocErrors};
update_docs(Db, Docs0, Options, interactive_edit) ->
- increment_stat(Db, {couchdb, database_writes}),
+ increment_stat(Db, [couchdb, database_writes]),
AllOrNothing = lists:member(all_or_nothing, Options),
Docs = tag_docs(Docs0),
@@ -960,6 +961,18 @@ set_commit_option(Options) ->
[full_commit|Options]
end.
+collect_results_with_metrics(Pid, MRef, []) ->
+ Begin = os:timestamp(),
+ try
+ collect_results(Pid, MRef, [])
+ after
+ ResultsTime = timer:now_diff(os:timestamp(), Begin) div 1000,
+ couch_stats:update_histogram(
+ [couchdb, collect_results_time],
+ ResultsTime
+ )
+ end.
+
collect_results(Pid, MRef, ResultsAcc) ->
receive
{result, Pid, Result} ->
@@ -981,7 +994,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1,
MRef = erlang:monitor(process, Pid),
try
Pid ! {update_docs, self(), DocBuckets, NonRepDocs, MergeConflicts, FullCommit},
- case collect_results(Pid, MRef, []) of
+ case collect_results_with_metrics(Pid, MRef, []) of
{ok, Results} -> {ok, Results};
retry ->
% This can happen if the db file we wrote to was swapped out by
@@ -995,7 +1008,7 @@ write_and_commit(#db{main_pid=Pid, user_ctx=Ctx}=Db, DocBuckets1,
DocBuckets3 = prepare_doc_summaries(Db2, DocBuckets2),
close(Db2),
Pid ! {update_docs, self(), DocBuckets3, NonRepDocs, MergeConflicts, FullCommit},
- case collect_results(Pid, MRef, []) of
+ case collect_results_with_metrics(Pid, MRef, []) of
{ok, Results} -> {ok, Results};
retry -> throw({update_error, compaction_retry})
end
@@ -1338,7 +1351,7 @@ increment_stat(#db{options = Options}, Stat) ->
true ->
ok;
false ->
- couch_stats_collector:increment(Stat)
+ couch_stats:increment_counter(Stat)
end.
skip_deleted(FoldFun) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_db_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl
index 2c1e808..cd434df 100644
--- a/src/couch_db_updater.erl
+++ b/src/couch_db_updater.erl
@@ -60,6 +60,7 @@ init({DbName, Filepath, Fd, Options}) ->
end
end,
Db = init_db(DbName, Filepath, Fd, Header, Options),
+ couch_stats_process_tracker:track([couchdb, open_databases]),
% we don't load validation funs here because the fabric query is liable to
% race conditions. Instead see couch_db:validate_doc_update, which loads
% them lazily
@@ -784,6 +785,16 @@ update_docs_int(Db, DocsList, NonRepDocs, MergeConflicts, FullCommit) ->
{ok, DocInfoByIdBTree2} = couch_btree:add_remove(DocInfoByIdBTree, IndexFullDocInfos, []),
{ok, DocInfoBySeqBTree2} = couch_btree:add_remove(DocInfoBySeqBTree, IndexFullDocInfos, RemoveSeqs),
+
+ WriteCount = length(IndexFullDocInfos),
+ couch_stats:increment_counter([couchdb, document_inserts],
+ WriteCount - length(RemoveSeqs)),
+ couch_stats:increment_counter([couchdb, document_writes], WriteCount),
+ couch_stats:increment_counter(
+ [couchdb, local_doc_writes],
+ length(NonRepDocs)
+ ),
+
Db3 = Db2#db{
id_tree = DocInfoByIdBTree2,
seq_tree = DocInfoBySeqBTree2,
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_file.erl
----------------------------------------------------------------------
diff --git a/src/couch_file.erl b/src/couch_file.erl
index 4fda603..be6b634 100644
--- a/src/couch_file.erl
+++ b/src/couch_file.erl
@@ -347,7 +347,7 @@ file_open_options(Options) ->
maybe_track_open_os_files(Options) ->
case not lists:member(sys_db, Options) of
true ->
- couch_stats_collector:track_process_count({couchdb, open_os_files});
+ couch_stats_process_tracker:track([couchdb, open_os_files]);
false ->
ok
end.
@@ -574,7 +574,7 @@ split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) ->
split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]).
-% System dbs aren't monitored by couch_stats_collector
+% System dbs aren't monitored by couch_stats_process_tracker
is_idle(#file{is_sys=true}) ->
case process_info(self(), monitored_by) of
{monitored_by, []} -> true;
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd.erl b/src/couch_httpd.erl
index 0c5af59..32950eb 100644
--- a/src/couch_httpd.erl
+++ b/src/couch_httpd.erl
@@ -350,8 +350,8 @@ handle_request_int(MochiReq, DefaultFun,
send_error(HttpReq, Error)
end,
RequestTime = round(timer:now_diff(os:timestamp(), Begin)/1000),
- couch_stats_collector:record({couchdb, request_time}, RequestTime),
- couch_stats_collector:increment({httpd, requests}),
+ couch_stats:update_histogram([couchdb, request_time], RequestTime),
+ couch_stats:increment_counter([httpd, requests]),
{ok, Resp}.
check_request_uri_length(Uri) ->
@@ -390,7 +390,7 @@ authenticate_request(Response, _AuthSrcs) ->
Response.
increment_method_stats(Method) ->
- couch_stats_collector:increment({httpd_request_methods, Method}).
+ couch_stats:increment_counter([httpd_request_methods, Method]).
validate_referer(Req) ->
Host = host_for_request(Req),
@@ -614,7 +614,7 @@ log_request(#httpd{mochi_req=MochiReq,peer=Peer}=Req, Code) ->
start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Length) ->
log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
+ couch_stats:increment_counter([httpd_status_codes, Code]),
Headers1 = Headers ++ server_header() ++
couch_httpd_auth:cookie_auth_header(Req, Headers),
Headers2 = couch_httpd_cors:cors_headers(Req, Headers1),
@@ -627,7 +627,7 @@ start_response_length(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Length) ->
start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) ->
log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
+ couch_stats:increment_counter([httpd_status_codes, Code]),
CookieHeader = couch_httpd_auth:cookie_auth_header(Req, Headers),
Headers1 = Headers ++ server_header() ++ CookieHeader,
Headers2 = couch_httpd_cors:cors_headers(Req, Headers1),
@@ -661,7 +661,7 @@ http_1_0_keep_alive(Req, Headers) ->
start_chunked_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) ->
log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
+ couch_stats:increment_counter([httpd_status_codes, Code]),
Headers1 = http_1_0_keep_alive(MochiReq, Headers),
Headers2 = Headers1 ++ server_header() ++
couch_httpd_auth:cookie_auth_header(Req, Headers1),
@@ -686,7 +686,7 @@ last_chunk(Resp) ->
send_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers, Body) ->
log_request(Req, Code),
- couch_stats_collector:increment({httpd_status_codes, Code}),
+ couch_stats:increment_counter([httpd_status_codes, Code]),
Headers1 = http_1_0_keep_alive(MochiReq, Headers),
if Code >= 500 ->
?LOG_ERROR("httpd ~p error response:~n ~s", [Code, Body]);
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_httpd_cors.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_cors.erl b/src/couch_httpd_cors.erl
index 15b838e..66878d0 100644
--- a/src/couch_httpd_cors.erl
+++ b/src/couch_httpd_cors.erl
@@ -174,7 +174,7 @@ handle_preflight_request(Origin, Host, MochiReq) ->
send_preflight_response(#httpd{mochi_req=MochiReq}=Req, Headers) ->
couch_httpd:log_request(Req, 204),
- couch_stats_collector:increment({httpd_status_codes, 204}),
+ couch_stats:increment_counter([httpd_status_codes, 204]),
Headers1 = couch_httpd:http_1_0_keep_alive(MochiReq, Headers),
Headers2 = Headers1 ++ couch_httpd:server_header() ++
couch_httpd_auth:cookie_auth_header(Req, Headers1),
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_httpd_db.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_db.erl b/src/couch_httpd_db.erl
index 77d8788..4fdc9aa 100644
--- a/src/couch_httpd_db.erl
+++ b/src/couch_httpd_db.erl
@@ -145,16 +145,8 @@ handle_changes_req2(Req, Db) ->
FeedChangesFun(MakeCallback(Resp))
end
end,
- couch_stats_collector:increment(
- {httpd, clients_requesting_changes}
- ),
- try
- WrapperFun(ChangesFun)
- after
- couch_stats_collector:decrement(
- {httpd, clients_requesting_changes}
- )
- end.
+ couch_stats_process_tracker:track([httpd, clients_requesting_changes]),
+ WrapperFun(ChangesFun).
handle_compact_req(#httpd{method='POST'}=Req, Db) ->
case Req#httpd.path_parts of
@@ -293,7 +285,7 @@ db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) ->
send_method_not_allowed(Req, "POST");
db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) ->
- couch_stats_collector:increment({httpd, bulk_requests}),
+ couch_stats:increment_counter([httpd, bulk_requests]),
couch_httpd:validate_ctype(Req, "application/json"),
{JsonProps} = couch_httpd:json_body_obj(Req),
case couch_util:get_value(<<"docs">>, JsonProps) of
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_httpd_stats_handlers.erl
----------------------------------------------------------------------
diff --git a/src/couch_httpd_stats_handlers.erl b/src/couch_httpd_stats_handlers.erl
index cd357ea..88376ab 100644
--- a/src/couch_httpd_stats_handlers.erl
+++ b/src/couch_httpd_stats_handlers.erl
@@ -11,46 +11,43 @@
% the License.
-module(couch_httpd_stats_handlers).
--include_lib("couch/include/couch_db.hrl").
+-include("couch_db.hrl").
-export([handle_stats_req/1]).
--import(couch_httpd, [
- send_json/2, send_json/3, send_json/4, send_method_not_allowed/2,
- start_json_response/2, send_chunk/2, end_json_response/1,
- start_chunked_response/3, send_error/4
-]).
handle_stats_req(#httpd{method='GET', path_parts=[_]}=Req) ->
- flush(Req),
- send_json(Req, couch_stats_aggregator:all(range(Req)));
-
-handle_stats_req(#httpd{method='GET', path_parts=[_, _Mod]}) ->
- throw({bad_request, <<"Stat names must have exactly two parts.">>});
-
-handle_stats_req(#httpd{method='GET', path_parts=[_, Mod, Key]}=Req) ->
- flush(Req),
- Stats = couch_stats_aggregator:get_json({list_to_atom(binary_to_list(Mod)),
- list_to_atom(binary_to_list(Key))}, range(Req)),
- send_json(Req, {[{Mod, {[{Key, Stats}]}}]});
-
-handle_stats_req(#httpd{method='GET', path_parts=[_, _Mod, _Key | _Extra]}) ->
- throw({bad_request, <<"Stat names must have exactly two parts.">>});
-
-handle_stats_req(Req) ->
- send_method_not_allowed(Req, "GET").
-
-range(Req) ->
- case couch_util:get_value("range", couch_httpd:qs(Req)) of
- undefined ->
- 0;
- Value ->
- list_to_integer(Value)
- end.
-
-flush(Req) ->
- case couch_util:get_value("flush", couch_httpd:qs(Req)) of
- "true" ->
- couch_stats_aggregator:collect_sample();
- _Else ->
- ok
- end.
+ Stats = couch_stats:fetch(),
+ Nested = nest(Stats),
+ EJSON = to_ejson(Nested),
+ couch_httpd:send_json(Req, EJSON).
+
+nest(Proplist) ->
+ nest(Proplist, []).
+
+nest([], Acc) ->
+ Acc;
+nest([{[Key|Keys], Value}|Rest], Acc) ->
+ Acc1 = case proplists:lookup(Key, Acc) of
+ {Key, Old} ->
+ [{Key, nest([{Keys, Value}], Old)}|proplists:delete(Key, Acc)];
+ none ->
+ Term = lists:foldr(fun(K, A) -> [{K, A}] end, Value, Keys),
+ [{Key, Term}|Acc]
+ end,
+ nest(Rest, Acc1).
+
+to_ejson([{_, _}|_]=Proplist) ->
+ EJSONProps = lists:map(
+ fun({Key, Value}) -> {maybe_format_key(Key), to_ejson(Value)} end,
+ Proplist
+ ),
+ {EJSONProps};
+to_ejson(NotAProplist) ->
+ NotAProplist.
+
+maybe_format_key(Key) when is_integer(Key) ->
+ maybe_format_key(integer_to_list(Key));
+maybe_format_key(Key) when is_list(Key) ->
+ list_to_binary(Key);
+maybe_format_key(Key) ->
+ Key.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_lru.erl
----------------------------------------------------------------------
diff --git a/src/couch_lru.erl b/src/couch_lru.erl
index ad432ec..d58eb69 100644
--- a/src/couch_lru.erl
+++ b/src/couch_lru.erl
@@ -52,6 +52,7 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
{gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)};
false ->
true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
+ couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
close_int(gb_trees:next(Iter), update(DbName, Cache))
end;
false ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_query_servers.erl
----------------------------------------------------------------------
diff --git a/src/couch_query_servers.erl b/src/couch_query_servers.erl
index c84ff7e..13b0b91 100644
--- a/src/couch_query_servers.erl
+++ b/src/couch_query_servers.erl
@@ -76,6 +76,7 @@ map_docs(Proc, Docs) ->
FunsResults)
end,
Docs),
+ couch_stats:increment_counter([couchdb, couchjs, map_doc], length(Docs)),
{ok, Results}.
map_doc_raw(Proc, Doc) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_server.erl b/src/couch_server.erl
index 964ecad..66a3c74 100644
--- a/src/couch_server.erl
+++ b/src/couch_server.erl
@@ -345,8 +345,8 @@ handle_call({open_result, DbName, {ok, Db}}, {FromPid, _Tag}, Server) ->
link(Db#db.main_pid),
true = ets:delete(couch_dbs_pid_to_name, FromPid),
case erase({async_open, DbName}) of undefined -> ok; T0 ->
- ?LOG_INFO("needed ~p ms to open new ~s", [timer:now_diff(os:timestamp(),T0)/1000,
- DbName])
+ OpenTime = timer:now_diff(os:timestamp(), T0) / 1000,
+ couch_stats:update_histogram([couchdb, db_open_time], OpenTime)
end,
% icky hack of field values - compactor_pid used to store clients
% and fd used to possibly store a creation request
@@ -363,8 +363,6 @@ handle_call({open_result, DbName, {ok, Db}}, {FromPid, _Tag}, Server) ->
true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}),
Lru = case couch_db:is_system_db(Db) of
false ->
- Stat = {couchdb, open_databases},
- couch_stats_collector:track_process_count(Db#db.main_pid, Stat),
couch_lru:insert(DbName, Server#server.lru);
true ->
Server#server.lru
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_stats_aggregator.erl
----------------------------------------------------------------------
diff --git a/src/couch_stats_aggregator.erl b/src/couch_stats_aggregator.erl
deleted file mode 100644
index 45987d6..0000000
--- a/src/couch_stats_aggregator.erl
+++ /dev/null
@@ -1,312 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_stats_aggregator).
--behaviour(gen_server).
--behaviour(config_listener).
-
--export([start/0, start/1, stop/0]).
--export([all/0, all/1, get/1, get/2, get_json/1, get_json/2, collect_sample/0]).
-
--export([init/1, terminate/2, code_change/3]).
--export([handle_call/3, handle_cast/2, handle_info/2]).
-
-% config_listener api
--export([handle_config_change/5]).
-
-
--record(aggregate, {
- description = <<"">>,
- seconds = 0,
- count = 0,
- current = null,
- sum = null,
- mean = null,
- variance = null,
- stddev = null,
- min = null,
- max = null,
- samples = []
-}).
-
-
-start() ->
- PrivDir = couch_util:priv_dir(),
- start(filename:join(PrivDir, "stat_descriptions.cfg")).
-
-start(FileName) ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [FileName], []).
-
-stop() ->
- gen_server:cast(?MODULE, stop).
-
-all() ->
- ?MODULE:all(0).
-all(Time) when is_binary(Time) ->
- ?MODULE:all(list_to_integer(binary_to_list(Time)));
-all(Time) when is_atom(Time) ->
- ?MODULE:all(list_to_integer(atom_to_list(Time)));
-all(Time) when is_integer(Time) ->
- Aggs = ets:match(?MODULE, {{'$1', Time}, '$2'}),
- Stats = lists:map(fun([Key, Agg]) -> {Key, Agg} end, Aggs),
- case Stats of
- [] ->
- {[]};
- _ ->
- Ret = lists:foldl(fun({{Mod, Key}, Agg}, Acc) ->
- CurrKeys = case proplists:lookup(Mod, Acc) of
- none -> [];
- {Mod, {Keys}} -> Keys
- end,
- NewMod = {[{Key, to_json_term(Agg)} | CurrKeys]},
- [{Mod, NewMod} | proplists:delete(Mod, Acc)]
- end, [], Stats),
- {Ret}
- end.
-
-get(Key) ->
- ?MODULE:get(Key, 0).
-get(Key, Time) when is_binary(Time) ->
- ?MODULE:get(Key, list_to_integer(binary_to_list(Time)));
-get(Key, Time) when is_atom(Time) ->
- ?MODULE:get(Key, list_to_integer(atom_to_list(Time)));
-get(Key, Time) when is_integer(Time) ->
- case ets:lookup(?MODULE, {make_key(Key), Time}) of
- [] -> #aggregate{seconds=Time};
- [{_, Agg}] -> Agg
- end.
-
-get_json(Key) ->
- get_json(Key, 0).
-get_json(Key, Time) ->
- to_json_term(?MODULE:get(Key, Time)).
-
-collect_sample() ->
- gen_server:call(?MODULE, collect_sample, infinity).
-
-
-init(StatDescsFileName) ->
- % Create an aggregate entry for each {description, rate} pair.
- ets:new(?MODULE, [named_table, set, protected]),
- SampleStr = config:get("stats", "samples", "[0]"),
- {ok, Samples} = couch_util:parse_term(SampleStr),
- {ok, Descs} = file:consult(StatDescsFileName),
- lists:foreach(fun({Sect, Key, Value}) ->
- lists:foreach(fun(Secs) ->
- Agg = #aggregate{
- description=list_to_binary(Value),
- seconds=Secs
- },
- ets:insert(?MODULE, {{{Sect, Key}, Secs}, Agg})
- end, Samples)
- end, Descs),
-
- ok = config:listen_for_changes(?MODULE, nil),
-
- Rate = list_to_integer(config:get("stats", "rate", "1000")),
- % TODO: Add timer_start to kernel start options.
- {ok, TRef} = timer:apply_after(Rate, ?MODULE, collect_sample, []),
- {ok, {TRef, Rate}}.
-
-terminate(_Reason, {TRef, _Rate}) ->
- timer:cancel(TRef),
- ok.
-
-handle_call(collect_sample, _, {OldTRef, SampleInterval}) ->
- timer:cancel(OldTRef),
- {ok, TRef} = timer:apply_after(SampleInterval, ?MODULE, collect_sample, []),
- % Gather new stats values to add.
- Incs = lists:map(fun({Key, Value}) ->
- {Key, {incremental, Value}}
- end, couch_stats_collector:all(incremental)),
- Abs = lists:map(fun({Key, Values}) ->
- couch_stats_collector:clear(Key),
- Values2 = case Values of
- X when is_list(X) -> X;
- Else -> [Else]
- end,
- {_, Mean} = lists:foldl(fun(Val, {Count, Curr}) ->
- {Count+1, Curr + (Val - Curr) / (Count+1)}
- end, {0, 0}, Values2),
- {Key, {absolute, Mean}}
- end, couch_stats_collector:all(absolute)),
-
- Values = Incs ++ Abs,
- Now = os:timestamp(),
- lists:foreach(fun({{Key, Rate}, Agg}) ->
- NewAgg = case proplists:lookup(Key, Values) of
- none ->
- rem_values(Now, Agg);
- {Key, {Type, Value}} ->
- NewValue = new_value(Type, Value, Agg#aggregate.current),
- Agg2 = add_value(Now, NewValue, Agg),
- rem_values(Now, Agg2)
- end,
- ets:insert(?MODULE, {{Key, Rate}, NewAgg})
- end, ets:tab2list(?MODULE)),
- {reply, ok, {TRef, SampleInterval}}.
-
-handle_cast(stop, State) ->
- {stop, normal, State}.
-
-handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
- erlang:send_after(5000, self(), restart_config_listener),
- {noreply, State};
-handle_info(restart_config_listener, State) ->
- ok = config:listen_for_changes(?MODULE, nil),
- {noreply, State};
-handle_info(_Info, State) ->
- {noreply, State}.
-
-code_change(_OldVersion, State, _Extra) ->
- {ok, State}.
-
-
-handle_config_change("stats", _, _, _, _) ->
- exit(whereis(?MODULE), config_change),
- remove_handler;
-handle_config_change(_, _, _, _, _) ->
- {ok, nil}.
-
-
-new_value(incremental, Value, null) ->
- Value;
-new_value(incremental, Value, Current) ->
- Value - Current;
-new_value(absolute, Value, _Current) ->
- Value.
-
-add_value(Time, Value, #aggregate{count=Count, seconds=Secs}=Agg) when Count < 1 ->
- Samples = case Secs of
- 0 -> [];
- _ -> [{Time, Value}]
- end,
- Agg#aggregate{
- count=1,
- current=Value,
- sum=Value,
- mean=Value,
- variance=0.0,
- stddev=null,
- min=Value,
- max=Value,
- samples=Samples
- };
-add_value(Time, Value, Agg) ->
- #aggregate{
- count=Count,
- current=Current,
- sum=Sum,
- mean=Mean,
- variance=Variance,
- samples=Samples
- } = Agg,
-
- NewCount = Count + 1,
- NewMean = Mean + (Value - Mean) / NewCount,
- NewVariance = Variance + (Value - Mean) * (Value - NewMean),
- StdDev = case NewCount > 1 of
- false -> null;
- _ -> math:sqrt(NewVariance / (NewCount - 1))
- end,
- Agg2 = Agg#aggregate{
- count=NewCount,
- current=Current + Value,
- sum=Sum + Value,
- mean=NewMean,
- variance=NewVariance,
- stddev=StdDev,
- min=lists:min([Agg#aggregate.min, Value]),
- max=lists:max([Agg#aggregate.max, Value])
- },
- case Agg2#aggregate.seconds of
- 0 -> Agg2;
- _ -> Agg2#aggregate{samples=[{Time, Value} | Samples]}
- end.
-
-rem_values(Time, Agg) ->
- Seconds = Agg#aggregate.seconds,
- Samples = Agg#aggregate.samples,
- Pred = fun({When, _Value}) ->
- timer:now_diff(Time, When) =< (Seconds * 1000000)
- end,
- {Keep, Remove} = lists:splitwith(Pred, Samples),
- Agg2 = lists:foldl(fun({_, Value}, Acc) ->
- rem_value(Value, Acc)
- end, Agg, Remove),
- Agg2#aggregate{samples=Keep}.
-
-rem_value(_Value, #aggregate{count=Count, seconds=Secs}) when Count =< 1 ->
- #aggregate{seconds=Secs};
-rem_value(Value, Agg) ->
- #aggregate{
- count=Count,
- sum=Sum,
- mean=Mean,
- variance=Variance
- } = Agg,
-
- OldMean = (Mean * Count - Value) / (Count - 1),
- OldVariance = Variance - (Value - OldMean) * (Value - Mean),
- OldCount = Count - 1,
- StdDev = case OldCount > 1 of
- false -> null;
- _ -> math:sqrt(clamp_value(OldVariance / (OldCount - 1)))
- end,
- Agg#aggregate{
- count=OldCount,
- sum=Sum-Value,
- mean=clamp_value(OldMean),
- variance=clamp_value(OldVariance),
- stddev=StdDev
- }.
-
-to_json_term(Agg) ->
- {Min, Max} = case Agg#aggregate.seconds > 0 of
- false ->
- {Agg#aggregate.min, Agg#aggregate.max};
- _ ->
- case length(Agg#aggregate.samples) > 0 of
- true ->
- Extract = fun({_Time, Value}) -> Value end,
- Samples = lists:map(Extract, Agg#aggregate.samples),
- {lists:min(Samples), lists:max(Samples)};
- _ ->
- {null, null}
- end
- end,
- {[
- {description, Agg#aggregate.description},
- {current, round_value(Agg#aggregate.sum)},
- {sum, round_value(Agg#aggregate.sum)},
- {mean, round_value(Agg#aggregate.mean)},
- {stddev, round_value(Agg#aggregate.stddev)},
- {min, Min},
- {max, Max}
- ]}.
-
-make_key({Mod, Val}) when is_integer(Val) ->
- {Mod, list_to_atom(integer_to_list(Val))};
-make_key(Key) ->
- Key.
-
-round_value(Val) when not is_number(Val) ->
- Val;
-round_value(Val) when Val == 0 ->
- Val;
-round_value(Val) ->
- erlang:round(Val * 1000.0) / 1000.0.
-
-clamp_value(Val) when Val > 0.00000000000001 ->
- Val;
-clamp_value(_) ->
- 0.0.
http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/62004cd9/src/couch_stats_collector.erl
----------------------------------------------------------------------
diff --git a/src/couch_stats_collector.erl b/src/couch_stats_collector.erl
deleted file mode 100644
index 5bf4864..0000000
--- a/src/couch_stats_collector.erl
+++ /dev/null
@@ -1,133 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% todo
-% - remove existance check on increment(), decrement() and record(). have
-% modules initialize counters on startup.
-
--module(couch_stats_collector).
-
--behaviour(gen_server).
--vsn(1).
-
--export([start/0, stop/0]).
--export([all/0, all/1, get/1, increment/1, decrement/1, record/2, clear/1]).
--export([track_process_count/1, track_process_count/2]).
-
--export([init/1, terminate/2, code_change/3]).
--export([handle_call/3, handle_cast/2, handle_info/2]).
-
--define(HIT_TABLE, stats_hit_table).
--define(ABS_TABLE, stats_abs_table).
-
-start() ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-stop() ->
- gen_server:call(?MODULE, stop).
-
-all() ->
- ets:tab2list(?HIT_TABLE) ++ abs_to_list().
-
-all(Type) ->
- case Type of
- incremental -> ets:tab2list(?HIT_TABLE);
- absolute -> abs_to_list()
- end.
-
-get(Key) ->
- case ets:lookup(?HIT_TABLE, Key) of
- [] ->
- case ets:lookup(?ABS_TABLE, Key) of
- [] ->
- nil;
- AbsVals ->
- lists:map(fun({_, Value}) -> Value end, AbsVals)
- end;
- [{_, Counter}] ->
- Counter
- end.
-
-increment(Key) ->
- Key2 = make_key(Key),
- case catch ets:update_counter(?HIT_TABLE, Key2, 1) of
- {'EXIT', {badarg, _}} ->
- catch ets:insert(?HIT_TABLE, {Key2, 1}),
- ok;
- _ ->
- ok
- end.
-
-decrement(Key) ->
- Key2 = make_key(Key),
- case catch ets:update_counter(?HIT_TABLE, Key2, -1) of
- {'EXIT', {badarg, _}} ->
- catch ets:insert(?HIT_TABLE, {Key2, -1}),
- ok;
- _ -> ok
- end.
-
-record(Key, Value) ->
- catch ets:insert(?ABS_TABLE, {make_key(Key), Value}).
-
-clear(Key) ->
- catch ets:delete(?ABS_TABLE, make_key(Key)).
-
-track_process_count(Stat) ->
- track_process_count(self(), Stat).
-
-track_process_count(Pid, Stat) ->
- ok = couch_stats_collector:increment(Stat),
- gen_server:cast(?MODULE, {track_process_count, Pid, Stat}).
-
-
-init(_) ->
- ets:new(?HIT_TABLE, [named_table, set, public]),
- ets:new(?ABS_TABLE, [named_table, duplicate_bag, public]),
- {ok, dict:new()}.
-
-terminate(_Reason, _State) ->
- ok.
-
-handle_call(stop, _, State) ->
- {stop, normal, stopped, State}.
-
-handle_cast({track_process_count, Pid, Stat}, State) ->
- Ref = erlang:monitor(process, Pid),
- {noreply, dict:store(Ref, Stat, State)}.
-
-handle_info({'DOWN', Ref, _, _, _}, State) ->
- Stat = dict:fetch(Ref, State),
- couch_stats_collector:decrement(Stat),
- {noreply, dict:erase(Ref, State)}.
-
-code_change(_, State, _Extra) ->
- {ok, State}.
-
-
-make_key({Module, Key}) when is_integer(Key) ->
- {Module, list_to_atom(integer_to_list(Key))};
-make_key(Key) ->
- Key.
-
-abs_to_list() ->
- SortedKVs = lists:sort(ets:tab2list(?ABS_TABLE)),
- lists:foldl(fun({Key, Val}, Acc) ->
- case Acc of
- [] ->
- [{Key, [Val]}];
- [{Key, Prev} | Rest] ->
- [{Key, [Val | Prev]} | Rest];
- Others ->
- [{Key, [Val]} | Others]
- end
- end, [], SortedKVs).