You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2020/01/07 08:01:40 UTC
[couchdb] branch mango_metrics updated (2671d20 -> 80322c8)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a change to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 2671d20 Refactor Mango warning generation
new 2b69e48 Refactor Mango warning generation
new 622fdb4 Add mango.query_invalid_index counter
new e8cd2aa Instrument Mango execution stats
new 80322c8 Warn on mango index scan
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2671d20)
\
N -- N -- N refs/heads/mango_metrics (80322c8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/priv/stats_descriptions.cfg | 24 ++++++++++++
src/mango/src/mango_cursor.erl | 67 +++++++++++++++++++++++----------
src/mango/src/mango_cursor_text.erl | 4 +-
src/mango/src/mango_cursor_view.erl | 4 +-
src/mango/src/mango_execution_stats.erl | 18 ++++++++-
src/mango/src/mango_httpd.erl | 2 +-
6 files changed, 92 insertions(+), 27 deletions(-)
[couchdb] 03/04: Instrument Mango execution stats
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e8cd2aa7709894169d397d916f1e87bec3c2628b
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 11:49:41 2020 +0000
Instrument Mango execution stats
Adds metrics for mango execution statistics:
* total docs examined (counter)
* total results returned (counter)
* query time (histogram)
---
src/couch/priv/stats_descriptions.cfg | 16 ++++++++++++++++
src/mango/src/mango_execution_stats.erl | 16 +++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index 0d81ea3..d333ea4 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -306,3 +306,19 @@
{type, counter},
{desc, <<"number of mango queries that generated an invalid index warning">>}
]}.
+{[mango, docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries">>}
+]}.
+{[mango, quorum_docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries, using cluster quorum">>}
+]}.
+{[mango, results_returned], [
+ {type, counter},
+ {desc, <<"number of rows returned by mango queries">>}
+]}.
+{[mango, query_time], [
+ {type, histogram},
+ {desc, <<"length of time processing a mango query">>}
+]}.
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd7..7f49f40 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -81,10 +81,24 @@ log_end(Stats) ->
}.
+incr_metrics(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount,
+ executionTimeMs = ExecTime
+ }) ->
+ couch_stats:increment_counter([mango, docs_examined], Docs),
+ couch_stats:increment_counter([mango, quorum_docs_examined], DocsQuorum),
+ couch_stats:increment_counter([mango, results_returned], ResultCount),
+ couch_stats:update_histogram([mango, query_time], ExecTime).
+
+
maybe_add_stats(Opts, UserFun, Stats, UserAcc) ->
+ Stats0 = log_end(Stats),
+ incr_metrics(Stats0),
+
case couch_util:get_value(execution_stats, Opts) of
true ->
- Stats0 = log_end(Stats),
JSONValue = to_json(Stats0),
Arg = {add_key, execution_stats, JSONValue},
{_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
[couchdb] 04/04: Warn on mango index scan
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 80322c88cafc31ea8d0f101b184f08b7f37ad0e5
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 13:22:07 2020 +0000
Warn on mango index scan
Adds a warning to the _find endpoint if the ratio of docs scanned:
results returned is higher than a configurable threshold (default
10). This warning was previously generated in Fauxton; moving
it to the server side allows us to expose it via _stats as well.
---
src/couch/priv/stats_descriptions.cfg | 4 ++++
src/mango/src/mango_cursor.erl | 29 +++++++++++++++++++++++++++--
src/mango/src/mango_cursor_text.erl | 2 +-
src/mango/src/mango_cursor_view.erl | 2 +-
4 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index d333ea4..f6a01b6 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -322,3 +322,7 @@
{type, histogram},
{desc, <<"length of time processing a mango query">>}
]}.
+{[mango, too_many_docs_scanned], [
+ {type, counter},
+ {desc, <<"number of mango queries with >10 docs scanned per result">>}
+]}.
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 03be4ee..9814e67 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,7 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
- maybe_add_warning/3
+ maybe_add_warning/4
]).
@@ -187,10 +187,35 @@ invalid_index_warning(Index, Opts) ->
end.
+% warn if a large number of documents needed to be scanned per result
+% returned, implying a lot of in-memory filtering
+index_scan_warning(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount
+ }) ->
+ % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
+ DocsScanned = Docs + DocsQuorum,
+ Ratio = case ResultCount of
+ 0 -> DocsScanned;
+ _ -> DocsScanned / ResultCount
+ end,
+ Threshold = config:get("mango", "index_scan_warning_threshold", 10),
+ case Ratio > Threshold of
+ true ->
+ couch_stats:increment_counter([mango, too_many_docs_scanned]),
+ Reason = <<"the number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>,
+ [Reason];
+ _ ->
+ []
+ end.
+
+
maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
W0 = invalid_index_warning(Index, Opts),
W1 = no_index_warning(Index),
- Warnings = lists:append([W0, W1]),
+ W2 = index_scan_warning(Stats),
+ Warnings = lists:append([W0, W1, W2]),
maybe_add_warning_int(Warnings, UserFun, UserAcc).
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 8938f35..9323bcf 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -126,7 +126,7 @@ execute(Cursor, UserFun, UserAcc) ->
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1}
end.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 9f5e6ec..ceda896 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -145,7 +145,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
Stats0 = LastCursor#cursor.execution_stats,
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1};
{error, Reason} ->
{error, Reason}
[couchdb] 02/04: Add mango.query_invalid_index counter
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 622fdb4e379eedf404f4a338d912cf7acebb8c65
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 09:30:55 2020 +0000
Add mango.query_invalid_index counter
Adds a metric to expose the number of Mango queries that could not
use the index specified in the _find query.
---
src/couch/priv/stats_descriptions.cfg | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index ae203bb..0d81ea3 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -302,3 +302,7 @@
{type, counter},
{desc, <<"number of mango queries that could not use an index">>}
]}.
+{[mango, query_invalid_index], [
+ {type, counter},
+ {desc, <<"number of mango queries that generated an invalid index warning">>}
+]}.
[couchdb] 01/04: Refactor Mango warning generation
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2b69e489aa52834e1ce2e54d7294f188805fdf72
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 09:22:56 2020 +0000
Refactor Mango warning generation
Refactor the code generating Mango warnings to (hopefully!)
improve readability.
This also moves the metric counter for unindexed queries such that it
gets incremented only when no index is used. Previously, it would
increment if *any* warning was generated (e.g. when Mango warned that
the user-specified index was not found/valid but fell back to a
different index).
The warning string returned now also contains all warnings generated
for the query, delimited via newlines - not just the first one.
---
src/mango/src/mango_cursor.erl | 60 +++++++++++++++++++++++--------------
src/mango/src/mango_cursor_view.erl | 10 +++----
2 files changed, 43 insertions(+), 27 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index dc2ee74..03be4ee 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -114,7 +114,7 @@ filter_indexes(Indexes0, DesignId, ViewName) ->
remove_indexes_with_partial_filter_selector(Indexes) ->
- FiltFun = fun(Idx) ->
+ FiltFun = fun(Idx) ->
case mango_idx:get_partial_filter_selector(Idx) of
undefined -> true;
_ -> false
@@ -146,44 +146,60 @@ group_indexes_by_type(Indexes) ->
end, ?CURSOR_MODULES).
-maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) ->
- NoIndexWarning = case Index#idx.type of
+% warn if the _all_docs index was used to fulfil a query
+no_index_warning(Index) ->
+ case Index#idx.type of
<<"special">> ->
- <<"no matching index found, create an index to optimize query time">>;
+ couch_stats:increment_counter([mango, unindexed_queries]),
+ [<<"no matching index found, create an index to optimize query time.">>];
_ ->
- ok
- end,
+ []
+ end.
- UseIndexInvalidWarning = case lists:keyfind(use_index, 1, Opts) of
- {use_index, []} ->
- NoIndexWarning;
+
+% warn if user specified an index which doesn't exist or isn't valid
+% for the selector.
+% In this scenario, Mango will ignore the index hint and auto-select an index.
+invalid_index_warning(Index, Opts) ->
+ case lists:keyfind(use_index, 1, Opts) of
{use_index, [DesignId]} ->
case filter_indexes([Index], DesignId) of
[] ->
- fmt("_design/~s was not used because it does not contain a valid index for this query.",
- [ddoc_name(DesignId)]);
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
+ [ddoc_name(DesignId)]),
+ [Reason];
_ ->
- NoIndexWarning
+ []
end;
{use_index, [DesignId, ViewName]} ->
case filter_indexes([Index], DesignId, ViewName) of
[] ->
- fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
- [ddoc_name(DesignId), ViewName]);
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
+ [ddoc_name(DesignId), ViewName]),
+ [Reason];
_ ->
- NoIndexWarning
- end
- end,
+ []
+ end;
+ _ ->
+ []
+ end.
+
- maybe_add_warning_int(UseIndexInvalidWarning, UserFun, UserAcc).
+maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
+ W0 = invalid_index_warning(Index, Opts),
+ W1 = no_index_warning(Index),
+ Warnings = lists:append([W0, W1]),
+ maybe_add_warning_int(Warnings, UserFun, UserAcc).
-maybe_add_warning_int(ok, _, UserAcc) ->
+maybe_add_warning_int([], _, UserAcc) ->
UserAcc;
-maybe_add_warning_int(Warning, UserFun, UserAcc) ->
- couch_stats:increment_counter([mango, unindexed_queries]),
- Arg = {add_key, warning, Warning},
+maybe_add_warning_int(Warnings, UserFun, UserAcc) ->
+ WarningStr = lists:join(<<"\n">>, Warnings),
+ Arg = {add_key, warning, WarningStr},
{_Go, UserAcc0} = UserFun(Arg, UserAcc),
UserAcc0.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 1c4b342..9f5e6ec 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -44,7 +44,7 @@ create(Db, Indexes, Selector, Opts) ->
Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
- Bookmark = couch_util:get_value(bookmark, Opts),
+ Bookmark = couch_util:get_value(bookmark, Opts),
{ok, #cursor{
db = Db,
@@ -124,7 +124,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
BaseArgs = base_args(Cursor),
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
+ Args = mango_json_bookmark:update_args(Bookmark, Args0),
UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
@@ -410,7 +410,7 @@ apply_opts([{_, _} | Rest], Args) ->
doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
+ Db = Cursor#cursor.db,
Opts = Cursor#cursor.opts,
ExecutionStats = Cursor#cursor.execution_stats,
Selector = Cursor#cursor.selector,
@@ -460,8 +460,8 @@ is_design_doc(RowProps) ->
update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
+ Id = couch_util:get_value(id, Props),
+ Key = couch_util:get_value(key, Props),
Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key