You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2018/03/12 22:26:47 UTC
[couchdb] 01/01: EXPERIMENT: enable couch_file transient stats
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch experiment-transient-stats
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 09c58ad5b7176888d5ac24e3d5fe222648505455
Author: Russell Branca <ch...@apache.org>
AuthorDate: Mon Mar 12 22:05:44 2018 +0000
EXPERIMENT: enable couch_file transient stats
---
src/chttpd/src/chttpd_misc.erl | 10 +++
src/couch/src/couch_file.erl | 85 +++++++++++++++++++++++---
src/couch_stats/src/couch_stats.erl | 4 ++
src/couch_stats/src/couch_stats_aggregator.erl | 23 ++++++-
4 files changed, 113 insertions(+), 9 deletions(-)
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 253da23..0f0072e 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -332,6 +332,16 @@ handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_stats">> | Path]}=
chttpd:send_json(Req, EJSON1);
handle_node_req(#httpd{path_parts=[_, _Node, <<"_stats">>]}=Req) ->
send_method_not_allowed(Req, "GET");
+handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_transient_stats">> | Path]}=Req) ->
+ flush(Node, Req),
+ Stats0 = call_node(Node, couch_stats, fetch_transient, []),
+ Stats = couch_stats_httpd:transform_stats(Stats0),
+ Nested = couch_stats_httpd:nest(Stats),
+ EJSON0 = couch_stats_httpd:to_ejson(Nested),
+ EJSON1 = couch_stats_httpd:extract_path(Path, EJSON0),
+ chttpd:send_json(Req, EJSON1);
+handle_node_req(#httpd{path_parts=[_, _Node, <<"_transient_stats">>]}=Req) ->
+ send_method_not_allowed(Req, "GET");
% GET /_node/$node/_system
handle_node_req(#httpd{method='GET', path_parts=[_, Node, <<"_system">>]}=Req) ->
Stats = call_node(Node, chttpd_misc, get_stats, []),
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index acd4fda..3e36285 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -129,7 +129,7 @@ append_term_md5(Fd, Term, Options) ->
append_binary(Fd, Bin) ->
ioq:call(Fd, {append_bin, assemble_file_chunk(Bin)}, erlang:get(io_priority)).
-
+
append_binary_md5(Fd, Bin) ->
ioq:call(Fd,
{append_bin, assemble_file_chunk(Bin, crypto:hash(md5, Bin))},
@@ -358,7 +358,7 @@ init({Filepath, Options, ReturnPid, Ref}) ->
Limit = get_pread_limit(),
IsSys = lists:member(sys_db, Options),
update_read_timestamp(),
- case lists:member(create, Options) of
+ InitStatus = case lists:member(create, Options) of
true ->
filelib:ensure_dir(Filepath),
case file:open(Filepath, OpenOptions) of
@@ -406,7 +406,12 @@ init({Filepath, Options, ReturnPid, Ref}) ->
Error ->
init_status_error(ReturnPid, Ref, Error)
end
- end.
+ end,
+ case InitStatus of
+ {ok, _} -> ok = create_couch_file_metrics(Filepath);
+ _ -> ok
+ end,
+ InitStatus.
file_open_options(Options) ->
[read, raw, binary] ++ case lists:member(read_only, Options) of
@@ -427,6 +432,19 @@ maybe_track_open_os_files(Options) ->
terminate(_Reason, #file{fd = nil}) ->
ok;
terminate(_Reason, #file{fd = Fd}) ->
+ Filepath = erlang:get(metrics_name),
+ Metrics = [
+ {counter, ["reads", "count"]},
+ {counter, ["reads", "bytes"]},
+ {histogram, ["reads", "latency"]},
+ {counter, ["writes", "count"]},
+ {counter, ["writes", "bytes"]},
+ {histogram, ["writes", "latency"]}
+ ],
+ lists:foreach(fun({_, Name0}) ->
+ Name = ["transient", "couch_file", Filepath | Name0],
+ couch_stats:delete(Name)
+ end, Metrics),
ok = file:close(Fd).
handle_call(Msg, From, File) when ?IS_OLD_STATE(File) ->
@@ -437,8 +455,9 @@ handle_call(close, _From, #file{fd=Fd}=File) ->
handle_call({pread_iolist, Pos}, _From, File) ->
update_read_timestamp(),
+ T0 = os:timestamp(),
{LenIolist, NextPos} = read_raw_iolist_int(File, Pos, 4),
- case iolist_to_binary(LenIolist) of
+ Res = case iolist_to_binary(LenIolist) of
<<1:1/integer,Len:31/integer>> -> % an MD5-prefixed term
{Md5AndIoList, _} = read_raw_iolist_int(File, NextPos, Len+16),
{Md5, IoList} = extract_md5(Md5AndIoList),
@@ -446,7 +465,11 @@ handle_call({pread_iolist, Pos}, _From, File) ->
<<0:1/integer,Len:31/integer>> ->
{Iolist, _} = read_raw_iolist_int(File, NextPos, Len),
{reply, {ok, Iolist, <<>>}, File}
- end;
+ end,
+ T1 = os:timestamp(),
+ Delta = timer:now_diff(T1, T0) div 1000,
+ ok = update_read_metrics(iolist_size(LenIolist), Delta),
+ Res;
handle_call(bytes, _From, #file{fd = Fd} = File) ->
{reply, file:position(Fd, eof), File};
@@ -472,14 +495,19 @@ handle_call({truncate, Pos}, _From, #file{fd=Fd}=File) ->
end;
handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
+ T0 = os:timestamp(),
Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin),
Size = iolist_size(Blocks),
- case file:write(Fd, Blocks) of
+ Res = case file:write(Fd, Blocks) of
ok ->
{reply, {ok, Pos, Size}, File#file{eof = Pos + Size}};
Error ->
{reply, Error, reset_eof(File)}
- end;
+ end,
+ T1 = os:timestamp(),
+ Delta = timer:now_diff(T1, T0) div 1000,
+ ok = update_write_metrics(Size, Delta),
+ Res;
handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) ->
BinSize = byte_size(Bin),
@@ -501,6 +529,19 @@ handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) ->
{reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}.
handle_cast(close, Fd) ->
+ Filepath = erlang:get(metrics_name),
+ Metrics = [
+ {counter, ["reads", "count"]},
+ {counter, ["reads", "bytes"]},
+ {histogram, ["reads", "latency"]},
+ {counter, ["writes", "count"]},
+ {counter, ["writes", "bytes"]},
+ {histogram, ["writes", "latency"]}
+ ],
+ lists:foreach(fun({_, Name0}) ->
+ Name = ["transient", "couch_file", Filepath | Name0],
+ couch_stats:delete(Name)
+ end, Metrics),
{stop,normal,Fd}.
code_change(_OldVsn, State, _Extra) ->
@@ -733,6 +774,36 @@ reset_eof(#file{} = File) ->
{ok, Eof} = file:position(File#file.fd, eof),
File#file{eof = Eof}.
+create_couch_file_metrics(Filepath0) ->
+ %% Drop .couch and .suffix
+ Filepath = filename:rootname(filename:rootname(Filepath0)),
+ erlang:put(metrics_name, Filepath),
+ Metrics = [
+ {counter, ["reads", "count"]},
+ {counter, ["reads", "bytes"]},
+ {histogram, ["reads", "latency"]},
+ {counter, ["writes", "count"]},
+ {counter, ["writes", "bytes"]},
+ {histogram, ["writes", "latency"]}
+ ],
+ lists:foreach(fun({Type, Name0}) ->
+ Name = ["transient", "couch_file", Filepath | Name0],
+ ok = couch_stats:new(Type, Name)
+ end, Metrics).
+
+update_read_metrics(Bytes, Delta) ->
+ update_metrics("reads", Bytes, Delta).
+
+update_write_metrics(Bytes, Delta) ->
+ update_metrics("writes", Bytes, Delta).
+
+update_metrics(Type, Bytes, Delta) ->
+ Name = erlang:get(metrics_name),
+ catch couch_stats:increment_counter(["transient", "couch_file", Name, Type, "count"]),
+ catch couch_stats:increment_counter(["transient", "couch_file", Name, Type, "bytes"], Bytes),
+ catch couch_stats:update_histogram(["transient", "couch_file", Name, Type, "latency"], Delta),
+ ok.
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
diff --git a/src/couch_stats/src/couch_stats.erl b/src/couch_stats/src/couch_stats.erl
index 4fde14a..fb5f07a 100644
--- a/src/couch_stats/src/couch_stats.erl
+++ b/src/couch_stats/src/couch_stats.erl
@@ -16,6 +16,7 @@
start/0,
stop/0,
fetch/0,
+ fetch_transient/0,
reload/0,
sample/1,
new/2,
@@ -45,6 +46,9 @@ stop() ->
fetch() ->
couch_stats_aggregator:fetch().
+fetch_transient() ->
+ couch_stats_aggregator:fetch_transient().
+
reload() ->
couch_stats_aggregator:reload().
diff --git a/src/couch_stats/src/couch_stats_aggregator.erl b/src/couch_stats/src/couch_stats_aggregator.erl
index 17bd6fc..163069a 100644
--- a/src/couch_stats/src/couch_stats_aggregator.erl
+++ b/src/couch_stats/src/couch_stats_aggregator.erl
@@ -16,6 +16,7 @@
-export([
fetch/0,
+ fetch_transient/0,
flush/0,
reload/0
]).
@@ -36,6 +37,7 @@
-record(st, {
descriptions,
stats,
+ tstats,
collect_timer,
reload_timer
}).
@@ -44,6 +46,10 @@ fetch() ->
{ok, Stats} = gen_server:call(?MODULE, fetch),
Stats.
+fetch_transient() ->
+ {ok, Stats} = gen_server:call(?MODULE, fetch_transient),
+ Stats.
+
flush() ->
gen_server:call(?MODULE, flush).
@@ -58,10 +64,12 @@ init([]) ->
Interval = config:get_integer("stats", "interval", ?DEFAULT_INTERVAL),
{ok, CT} = timer:send_interval(Interval * 1000, self(), collect),
{ok, RT} = timer:send_interval(?RELOAD_INTERVAL * 1000, self(), reload),
- {ok, #st{descriptions=Descs, stats=[], collect_timer=CT, reload_timer=RT}}.
+ {ok, #st{descriptions=Descs, stats=[], tstats=[], collect_timer=CT, reload_timer=RT}}.
handle_call(fetch, _from, #st{stats = Stats}=State) ->
{reply, {ok, Stats}, State};
+handle_call(fetch_transient, _from, #st{tstats = TStats}=State) ->
+ {reply, {ok, TStats}, State};
handle_call(flush, _From, State) ->
{reply, ok, collect(State)};
handle_call(reload, _from, State) ->
@@ -147,4 +155,15 @@ collect(State) ->
end,
State#st.descriptions
),
- State#st{stats=Stats}.
+ TStats = lists:foldl(
+ fun
+ ({["transient" | _] = Name, Props0}, Acc) ->
+ Props = proplists:delete(tags, Props0),
+ [{Name, [{value, couch_stats:sample(Name)}|Props]} | Acc];
+ (_, Acc) ->
+ Acc
+ end,
+ [],
+ folsom_metrics:get_metrics_info()
+ ),
+ State#st{stats=Stats, tstats=TStats}.
--
To stop receiving notification emails like this one, please contact
chewbranca@apache.org.