You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2020/01/07 09:12:58 UTC
[couchdb] branch mango_metrics updated (80322c8 -> 35cbc7a)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a change to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 80322c8 Warn on mango index scan
discard e8cd2aa Instrument Mango execution stats
discard 622fdb4 Add mango.query_invalid_index counter
discard 2b69e48 Refactor Mango warning generation
new f85ae8b Refactor Mango warning generation
new 91e9317 Add mango.query_invalid_index counter
new 6b1b82d Instrument Mango execution stats
new 35cbc7a Warn on mango index scan
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (80322c8)
\
N -- N -- N refs/heads/mango_metrics (35cbc7a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rel/overlay/etc/default.ini | 20 ++++++----
src/mango/src/mango_cursor.erl | 88 +++++++++++++++++++++---------------------
2 files changed, 57 insertions(+), 51 deletions(-)
[couchdb] 04/04: Warn on mango index scan
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 35cbc7a604e426ff5579eb9fe9e5bad6abfe8b2e
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 13:22:07 2020 +0000
Warn on mango index scan
Adds a warning to the _find endpoint if the ratio of docs scanned:
results returned is higher than a configurable threshold (default
10). This warning was previously generated in Fauxton; moving
it to the server side allows us to expose it via _stats as well.
---
rel/overlay/etc/default.ini | 20 ++++++++++++--------
src/couch/priv/stats_descriptions.cfg | 4 ++++
src/mango/src/mango_cursor.erl | 34 +++++++++++++++++++++++++++++++---
src/mango/src/mango_cursor_text.erl | 2 +-
src/mango/src/mango_cursor_view.erl | 2 +-
5 files changed, 49 insertions(+), 13 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index a1df080..ab67ec8 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -167,8 +167,8 @@ enable_xframe_options = false
; CouchDB can optionally enforce a maximum uri length;
; max_uri_length = 8000
; changes_timeout = 60000
-; config_whitelist =
-; max_uri_length =
+; config_whitelist =
+; max_uri_length =
; rewrite_limit = 100
; x_forwarded_host = X-Forwarded-Host
; x_forwarded_proto = X-Forwarded-Proto
@@ -177,7 +177,7 @@ enable_xframe_options = false
max_http_request_size = 4294967296 ; 4GB
; [httpd_design_handlers]
-; _view =
+; _view =
; [ioq]
; concurrency = 10
@@ -191,7 +191,7 @@ port = 6984
; [chttpd_auth_cache]
; max_lifetime = 600000
-; max_objects =
+; max_objects =
; max_size = 104857600
; [mem3]
@@ -202,7 +202,7 @@ port = 6984
; [fabric]
; all_docs_concurrency = 10
-; changes_duration =
+; changes_duration =
; shard_timeout_factor = 2
; uuid_prefix_len = 7
; request_timeout = 60000
@@ -240,7 +240,7 @@ iterations = 10 ; iterations for password hashing
; proxy_use_secret = false
; comma-separated list of public fields, 404 if empty
; public_fields =
-; secret =
+; secret =
; users_db_public = false
; cookie_domain = example.com
@@ -313,12 +313,16 @@ os_process_limit = 100
;index_all_disabled = false
; Default limit value for mango _find queries.
;default_limit = 25
+; Ratio between documents scanned and results matched that will
+; generate a warning in the _find response. Setting this to 0 disables
+; the warning.
+;index_scan_warning_threshold = 10
[indexers]
couch_mrview = true
[feature_flags]
-; This enables any database to be created as a partitioned databases (except system db's).
+; This enables any database to be created as a partitioned databases (except system db's).
; Setting this to false will stop the creation of paritioned databases.
; paritioned||allowed* = true will scope the creation of partitioned databases
; to databases with 'allowed' prefix.
@@ -527,7 +531,7 @@ min_priority = 2.0
; The default number of results returned from a search on a partition
; of a database.
; limit_partitions = 2000
-
+
; The maximum number of results that can be returned from a global
; search query (or any search query on a database without user-defined
; partitions). Attempts to set ?limit=N higher than this value will
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index d333ea4..f6a01b6 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -322,3 +322,7 @@
{type, histogram},
{desc, <<"length of time processing a mango query">>}
]}.
+{[mango, too_many_docs_scanned], [
+ {type, counter},
+ {desc, <<"number of mango queries with >10 docs scanned per result">>}
+]}.
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 6161270..d0b44c1 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,7 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
- maybe_add_warning/3
+ maybe_add_warning/4
]).
@@ -186,16 +186,44 @@ invalid_index_warning_int(_, _) ->
[].
+% warn if a large number of documents needed to be scanned per result
+% returned, implying a lot of in-memory filtering
+index_scan_warning(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount
+ }) ->
+ % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
+ DocsScanned = Docs + DocsQuorum,
+ Ratio = index_scan_ratio(DocsScanned, ResultCount),
+ Threshold = config:get("mango", "index_scan_warning_threshold", 10),
+ index_scan_warning_int(Threshold, Ratio).
+
+
+index_scan_ratio(DocsScanned, 0) ->
+ DocsScanned;
+index_scan_ratio(DocsScanned, ResultCount) ->
+ DocsScanned / ResultCount.
+
+
+index_scan_warning_int(Threshold, Ratio) when is_integer(Threshold), Threshold > 0, Ratio > Threshold ->
+ couch_stats:increment_counter([mango, too_many_docs_scanned]),
+ Reason = <<"the number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>,
+ [Reason];
+index_scan_warning_int(_, _) ->
+ [].
+
+
maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
W0 = invalid_index_warning(Index, Opts),
W1 = no_index_warning(Index),
- Warnings = lists:append([W0, W1]),
+ W2 = index_scan_warning(Stats),
+ Warnings = lists:append([W0, W1, W2]),
maybe_add_warning_int(Warnings, UserFun, UserAcc).
maybe_add_warning_int([], _, UserAcc) ->
UserAcc;
-
maybe_add_warning_int(Warnings, UserFun, UserAcc) ->
WarningStr = lists:join(<<"\n">>, Warnings),
Arg = {add_key, warning, WarningStr},
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 8938f35..9323bcf 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -126,7 +126,7 @@ execute(Cursor, UserFun, UserAcc) ->
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1}
end.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 9f5e6ec..ceda896 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -145,7 +145,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
Stats0 = LastCursor#cursor.execution_stats,
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1};
{error, Reason} ->
{error, Reason}
[couchdb] 03/04: Instrument Mango execution stats
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6b1b82d121fc3974d564d53bc2efbb0db8d53406
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 11:49:41 2020 +0000
Instrument Mango execution stats
Adds metrics for mango execution statistics:
* total docs examined (counter)
* total results returned (counter)
* query time (histogram)
---
src/couch/priv/stats_descriptions.cfg | 16 ++++++++++++++++
src/mango/src/mango_execution_stats.erl | 16 +++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index 0d81ea3..d333ea4 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -306,3 +306,19 @@
{type, counter},
{desc, <<"number of mango queries that generated an invalid index warning">>}
]}.
+{[mango, docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries">>}
+]}.
+{[mango, quorum_docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries, using cluster quorum">>}
+]}.
+{[mango, results_returned], [
+ {type, counter},
+ {desc, <<"number of rows returned by mango queries">>}
+]}.
+{[mango, query_time], [
+ {type, histogram},
+ {desc, <<"length of time processing a mango query">>}
+]}.
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd7..7f49f40 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -81,10 +81,24 @@ log_end(Stats) ->
}.
+incr_metrics(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount,
+ executionTimeMs = ExecTime
+ }) ->
+ couch_stats:increment_counter([mango, docs_examined], Docs),
+ couch_stats:increment_counter([mango, quorum_docs_examined], DocsQuorum),
+ couch_stats:increment_counter([mango, results_returned], ResultCount),
+ couch_stats:update_histogram([mango, query_time], ExecTime).
+
+
maybe_add_stats(Opts, UserFun, Stats, UserAcc) ->
+ Stats0 = log_end(Stats),
+ incr_metrics(Stats0),
+
case couch_util:get_value(execution_stats, Opts) of
true ->
- Stats0 = log_end(Stats),
JSONValue = to_json(Stats0),
Arg = {add_key, execution_stats, JSONValue},
{_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
[couchdb] 01/04: Refactor Mango warning generation
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f85ae8bf767006b2898fbe412365823ae07fbf4a
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 09:22:56 2020 +0000
Refactor Mango warning generation
Refactor the code generating Mango warnings to (hopefully!)
improve readability.
This also moves the metric counter for unindexed queries such that it
gets incremented only when no index is used. Previously, it would
increment if *any* warning was generated (e.g. when Mango warned that
the user-specified index was not found/valid but fell back to a
different index).
The warning string returned now also contains all warnings generated
for the query, delimited via newlines - not just the first one.
---
src/mango/src/mango_cursor.erl | 77 ++++++++++++++++++++++---------------
src/mango/src/mango_cursor_view.erl | 10 ++---
2 files changed, 50 insertions(+), 37 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index dc2ee74..5f9690f 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -114,7 +114,7 @@ filter_indexes(Indexes0, DesignId, ViewName) ->
remove_indexes_with_partial_filter_selector(Indexes) ->
- FiltFun = fun(Idx) ->
+ FiltFun = fun(Idx) ->
case mango_idx:get_partial_filter_selector(Idx) of
undefined -> true;
_ -> false
@@ -146,44 +146,57 @@ group_indexes_by_type(Indexes) ->
end, ?CURSOR_MODULES).
-maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) ->
- NoIndexWarning = case Index#idx.type of
- <<"special">> ->
- <<"no matching index found, create an index to optimize query time">>;
+% warn if the _all_docs index was used to fulfil a query
+no_index_warning(#idx{type = Type}) when Type =:= <<"special">> ->
+ couch_stats:increment_counter([mango, unindexed_queries]),
+ [<<"no matching index found, create an index to optimize query time.">>];
+no_index_warning(_) ->
+ [].
+
+
+% warn if user specified an index which doesn't exist or isn't valid
+% for the selector.
+% In this scenario, Mango will ignore the index hint and auto-select an index.
+invalid_index_warning(Index, Opts) ->
+ UseIndex = lists:keyfind(use_index, 1, Opts),
+ invalid_index_warning_int(Index, UseIndex).
+
+
+invalid_index_warning_int(Index, {use_index, [DesignId]}) ->
+ case filter_indexes([Index], DesignId) of
+ [] ->
+ Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
+ [ddoc_name(DesignId)]),
+ [Reason];
_ ->
- ok
- end,
+ []
+ end;
+invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) ->
+ case filter_indexes([Index], DesignId, ViewName) of
+ [] ->
+ Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
+ [ddoc_name(DesignId), ViewName]),
+ [Reason];
+ _ ->
+ []
+ end;
+invalid_index_warning_int(_, _) ->
+ [].
- UseIndexInvalidWarning = case lists:keyfind(use_index, 1, Opts) of
- {use_index, []} ->
- NoIndexWarning;
- {use_index, [DesignId]} ->
- case filter_indexes([Index], DesignId) of
- [] ->
- fmt("_design/~s was not used because it does not contain a valid index for this query.",
- [ddoc_name(DesignId)]);
- _ ->
- NoIndexWarning
- end;
- {use_index, [DesignId, ViewName]} ->
- case filter_indexes([Index], DesignId, ViewName) of
- [] ->
- fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
- [ddoc_name(DesignId), ViewName]);
- _ ->
- NoIndexWarning
- end
- end,
- maybe_add_warning_int(UseIndexInvalidWarning, UserFun, UserAcc).
+maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
+ W0 = invalid_index_warning(Index, Opts),
+ W1 = no_index_warning(Index),
+ Warnings = lists:append([W0, W1]),
+ maybe_add_warning_int(Warnings, UserFun, UserAcc).
-maybe_add_warning_int(ok, _, UserAcc) ->
+maybe_add_warning_int([], _, UserAcc) ->
UserAcc;
-maybe_add_warning_int(Warning, UserFun, UserAcc) ->
- couch_stats:increment_counter([mango, unindexed_queries]),
- Arg = {add_key, warning, Warning},
+maybe_add_warning_int(Warnings, UserFun, UserAcc) ->
+ WarningStr = lists:join(<<"\n">>, Warnings),
+ Arg = {add_key, warning, WarningStr},
{_Go, UserAcc0} = UserFun(Arg, UserAcc),
UserAcc0.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 1c4b342..9f5e6ec 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -44,7 +44,7 @@ create(Db, Indexes, Selector, Opts) ->
Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
- Bookmark = couch_util:get_value(bookmark, Opts),
+ Bookmark = couch_util:get_value(bookmark, Opts),
{ok, #cursor{
db = Db,
@@ -124,7 +124,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
BaseArgs = base_args(Cursor),
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
+ Args = mango_json_bookmark:update_args(Bookmark, Args0),
UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
@@ -410,7 +410,7 @@ apply_opts([{_, _} | Rest], Args) ->
doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
+ Db = Cursor#cursor.db,
Opts = Cursor#cursor.opts,
ExecutionStats = Cursor#cursor.execution_stats,
Selector = Cursor#cursor.selector,
@@ -460,8 +460,8 @@ is_design_doc(RowProps) ->
update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
+ Id = couch_util:get_value(id, Props),
+ Key = couch_util:get_value(key, Props),
Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key
[couchdb] 02/04: Add mango.query_invalid_index counter
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch mango_metrics
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 91e9317aabce15752cb7c5520b954c351019f165
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Jan 6 09:30:55 2020 +0000
Add mango.query_invalid_index counter
Adds a metric to expose the number of Mango queries that could not
use the index specified in the _find query.
---
src/couch/priv/stats_descriptions.cfg | 4 ++++
src/mango/src/mango_cursor.erl | 2 ++
2 files changed, 6 insertions(+)
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index ae203bb..0d81ea3 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -302,3 +302,7 @@
{type, counter},
{desc, <<"number of mango queries that could not use an index">>}
]}.
+{[mango, query_invalid_index], [
+ {type, counter},
+ {desc, <<"number of mango queries that generated an invalid index warning">>}
+]}.
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 5f9690f..6161270 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -165,6 +165,7 @@ invalid_index_warning(Index, Opts) ->
invalid_index_warning_int(Index, {use_index, [DesignId]}) ->
case filter_indexes([Index], DesignId) of
[] ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
[ddoc_name(DesignId)]),
[Reason];
@@ -174,6 +175,7 @@ invalid_index_warning_int(Index, {use_index, [DesignId]}) ->
invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) ->
case filter_indexes([Index], DesignId, ViewName) of
[] ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
[ddoc_name(DesignId), ViewName]),
[Reason];