You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2025/03/22 02:12:23 UTC
(couchdb) branch couch-stats-resource-tracker-v3 updated: Add dbnames io matchers and tests
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch couch-stats-resource-tracker-v3
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/couch-stats-resource-tracker-v3 by this push:
new b91b89d87 Add dbnames io matchers and tests
b91b89d87 is described below
commit b91b89d87d6a3d83f82268ae8da801d43bcf6b14
Author: Russell Branca <ch...@apache.org>
AuthorDate: Fri Mar 21 19:12:14 2025 -0700
Add dbnames io matchers and tests
---
rel/overlay/etc/default.ini | 11 +++++++
src/couch_stats/src/csrt_logger.erl | 33 +++++++++++++++++--
src/couch_stats/test/eunit/csrt_logger_tests.erl | 42 ++++++++++++++++++++++++
3 files changed, 84 insertions(+), 2 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index aac3a5157..7e5a15ceb 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -1078,3 +1078,14 @@ fabric_rpc__get_all_security = true
fabric_rpc__open_doc = true
fabric_rpc__update_docs = true
fabric_rpc__open_shard = true
+
+;; CSRT dbname matchers
+;; Given a dbname and a positive integer, this will enable an IO matcher
+;; against the provided db for any requests that induce IO in quantities
+;; greater than the provided threshold on any one of: ioq_calls, rows_read
+;; docs_read, get_kp_node, get_kv_node, or changes_processed.
+;;
+;; foo = 100
+;; dbs = 123
+;; foo/bar = 200
+[csrt_logger.dbnames_io]
diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl
index 4059efdaf..b6e60dae7 100644
--- a/src/couch_stats/src/csrt_logger.erl
+++ b/src/couch_stats/src/csrt_logger.erl
@@ -68,6 +68,7 @@
-define(MATCHERS_KEY, {?MODULE, all_csrt_matchers}).
-define(CONF_MATCHERS_ENABLED, "csrt_logger.matchers_enabled").
-define(CONF_MATCHERS_THRESHOLD, "csrt_logger.matchers_threshold").
+-define(CONF_MATCHERS_DBNAMES, "csrt_logger.dbnames_io").
-record(st, {
matchers = #{}
@@ -241,6 +242,10 @@ matcher_on_dbname(DbName)
when is_binary(DbName) ->
ets:fun2ms(fun(#rctx{dbname=DbName1} = R) when DbName =:= DbName1 -> R end).
+matcher_on_dbname_io_threshold(DbName, Threshold)
+ when is_binary(DbName) ->
+ ets:fun2ms(fun(#rctx{dbname=DbName1, ioq_calls=IOQ, get_kv_node=KVN, get_kp_node=KPN, docs_read=Docs, rows_read=Rows, changes_processed=Chgs} = R) when DbName =:= DbName1 andalso ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or (Rows >= Threshold) or (Chgs >= Threshold)) -> R end).
+
-spec matcher_on_docs_read(Threshold :: pos_integer()) -> ets:match_spec().
matcher_on_docs_read(Threshold)
when is_integer(Threshold) andalso Threshold > 0 ->
@@ -331,8 +336,32 @@ initialize_matchers() ->
#{},
DefaultMatchers
),
- couch_log:notice("Initialized ~p CSRT Logger matchers", [maps:size(Matchers)]),
- persistent_term:put(?MATCHERS_KEY, Matchers),
+ Matchers1 = lists:foldl(
+ fun({Dbname, Value}, Matchers0) ->
+ try list_to_integer(Value) of
+ Threshold when Threshold > 0 ->
+ Name = "dbname_io__" ++ Dbname ++ "__" ++ Value,
+ DbnameB = list_to_binary(Dbname),
+ MSpec = matcher_on_dbname_io_threshold(DbnameB, Threshold),
+ case add_matcher(Name, MSpec, Matchers0) of
+ {ok, Matchers1} ->
+ Matchers1;
+ {error, badarg} ->
+ couch_log:warning("[~p] Failed to initialize matcher: ~p", [?MODULE, Name]),
+ Matchers0
+ end;
+ _ ->
+ Matchers0
+ catch error:badarg ->
+ couch_log:warning("[~p] Failed to initialize dbname io matcher on: ~p", [?MODULE, Dbname])
+ end
+ end,
+ Matchers,
+ config:get(?CONF_MATCHERS_DBNAMES)
+ ),
+
+ couch_log:notice("Initialized ~p CSRT Logger matchers", [maps:size(Matchers1)]),
+ persistent_term:put(?MATCHERS_KEY, Matchers1),
ok.
-spec matcher_enabled(Name :: string()) -> boolean().
diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl
index 47393052e..648b7601c 100644
--- a/src/couch_stats/test/eunit/csrt_logger_tests.erl
+++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl
@@ -24,6 +24,7 @@
-define(RCTX_COORDINATOR, {coordinator, foo, bar, 'GET', "/foo/_all_docs"}).
-define(THRESHOLD_DBNAME, <<"foo">>).
+-define(THRESHOLD_DBNAME_IO, 91).
-define(THRESHOLD_DOCS_READ, 123).
-define(THRESHOLD_IOQ_CALLS, 439).
-define(THRESHOLD_ROWS_READ, 143).
@@ -49,6 +50,7 @@ csrt_logger_matchers_test_() ->
fun teardown/1,
[
?TDEF_FE(t_matcher_on_dbname),
+ ?TDEF_FE(t_matcher_on_dbnames_io),
?TDEF_FE(t_matcher_on_docs_read),
?TDEF_FE(t_matcher_on_docs_written),
?TDEF_FE(t_matcher_on_rows_read),
@@ -87,6 +89,9 @@ setup() ->
ok = config:set("csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false),
ok = config:set("csrt_logger.matchers_threshold", "rows_read", integer_to_list(?THRESHOLD_ROWS_READ), false),
ok = config:set("csrt_logger.matchers_threshold", "worker_changes_processed", integer_to_list(?THRESHOLD_CHANGES), false),
+ ok = config:set("csrt_logger.dbnames_io", "foo", integer_to_list(?THRESHOLD_DBNAME_IO), false),
+ ok = config:set("csrt_logger.dbnames_io", "bar", integer_to_list(?THRESHOLD_DBNAME_IO), false),
+ ok = config:set("csrt_logger.dbnames_io", "foo/bar", integer_to_list(?THRESHOLD_DBNAME_IO), false),
csrt_logger:reload_matchers(),
#{ctx => Ctx, dbname => DbName, rctx => Rctx, rctxs => rctxs()}.
@@ -267,6 +272,34 @@ t_matcher_on_nonce(#{rctxs := Rctxs0}) ->
"Rows read matcher"
).
+t_matcher_on_dbnames_io(#{rctxs := Rctxs0}) ->
+ Threshold = ?THRESHOLD_DBNAME_IO,
+ SThreshold = integer_to_list(Threshold),
+ DbFoo = "foo",
+ DbBar = "bar",
+ MatcherFoo = matcher_for_csrt("dbname_io__" ++ DbFoo ++ "__" ++ SThreshold),
+ MatcherBar = matcher_for_csrt("dbname_io__" ++ DbBar ++ "__" ++ SThreshold),
+ MatcherFooBar = matcher_for_csrt("dbname_io__foo/bar__" ++ SThreshold),
+ %% Add an extra Rctx with dbname foo/bar to ensure correct naming matches
+ ExtraRctx = rctx_gen(#{dbname => <<"foo/bar">>, get_kp_node => Threshold + 10}),
+ %% Make sure we have at least one match
+ Rctxs = [ExtraRctx, rctx_gen(#{ioq_calls => Threshold + 10}) | Rctxs0],
+ ?assertEqual(
+ lists:sort(lists:filter(matcher_for_dbname_io(DbFoo, Threshold), Rctxs)),
+ lists:sort(lists:filter(MatcherFoo, Rctxs)),
+ "dbname_io foo matcher"
+ ),
+ ?assertEqual(
+ lists:sort(lists:filter(matcher_for_dbname_io(DbBar, Threshold), Rctxs)),
+ lists:sort(lists:filter(MatcherBar, Rctxs)),
+ "dbname_io bar matcher"
+ ),
+ ?assertEqual(
+ [ExtraRctx],
+ lists:sort(lists:filter(MatcherFooBar, Rctxs)),
+ "dbname_io foo/bar matcher"
+ ).
+
load_rctx(PidRef) ->
timer:sleep(50), %% Add slight delay to accumulate RPC response deltas
csrt:get_resource(PidRef).
@@ -289,6 +322,15 @@ matcher_for_csrt(MatcherName) ->
Matchers = #{MatcherName => {_, _} = csrt_logger:get_matcher(MatcherName)},
fun(Rctx) -> csrt_logger:is_match(Rctx, Matchers) end.
+matcher_for_dbname_io(Dbname0, Threshold) ->
+ Dbname = list_to_binary(Dbname0),
+ fun(Rctx) ->
+ DbnameA = csrt_util:field(dbname, Rctx),
+ Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read, changes_processed],
+ Vals = [{F, csrt_util:field(F, Rctx)} || F <- Fields],
+ Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun(V) -> V >= Threshold end, Vals)
+ end.
+
nonce() ->
couch_util:to_hex(crypto:strong_rand_bytes(5)).