You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2020/01/07 15:32:20 UTC
[couchdb] branch fix_mango_execution_stats created (now 9213fd0)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a change to branch fix_mango_execution_stats
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 9213fd0 Fix missing mango execution stats
This branch includes the following new commits:
new 9213fd0 Fix missing mango execution stats
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Fix missing mango execution stats
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch fix_mango_execution_stats
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9213fd0e7db03a49ca42ab03ea2a8393869664e1
Author: Will Holley <wi...@gmail.com>
AuthorDate: Tue Jan 7 13:40:32 2020 +0000
Fix missing mango execution stats
The previous implementation of Mango execution stats relied on
passing the docs_examined count from each shard to the coordinator
in the view_row record. This failed to collect the count of
documents read which weren't followed by a match (in a given shard).
For example, if an index was scanned but no documents were matched,
the docs_examined would be 0, when it should be equal to the number
of documents in the index.
This commit changes the implementation so that docs examined is passed
only when each shard has completed its index scan. This requires a new
handler in fabric so, in it's current state, will generate errors
in mixed-version clusters.
---
src/fabric/src/fabric_view_all_docs.erl | 7 +++-
src/fabric/src/fabric_view_map.erl | 6 +++
src/mango/src/mango_cursor_view.erl | 65 ++++++++++++++-----------------
src/mango/test/15-execution-stats-test.py | 4 ++
4 files changed, 45 insertions(+), 37 deletions(-)
diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl
index 07cd1b1..e4d3d4a 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -239,8 +239,13 @@ handle_message(#view_row{} = Row, {Worker, From}, State) ->
handle_message(complete, Worker, State) ->
Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters),
- fabric_view:maybe_send_row(State#collector{counters = Counters}).
+ fabric_view:maybe_send_row(State#collector{counters = Counters});
+handle_message({execution_stats, _} = Msg, {_,From}, St) ->
+ #collector{callback=Callback, user_acc=AccIn} = St,
+ {Go, Acc} = Callback(Msg, AccIn),
+ rexi:stream_ack(From),
+ {Go, St#collector{user_acc=Acc}}.
merge_row(fwd, Row, Rows) ->
lists:keymerge(#view_row.id, [Row], Rows);
diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl
index 5a5cc13..b8d0d39 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -180,6 +180,12 @@ handle_message(complete, Worker, State) ->
Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters),
fabric_view:maybe_send_row(State#collector{counters = Counters});
+handle_message({execution_stats, _} = Msg, {_,From}, St) ->
+ #collector{callback=Callback, user_acc=AccIn} = St,
+ {Go, Acc} = Callback(Msg, AccIn),
+ rexi:stream_ack(From),
+ {Go, St#collector{user_acc=Acc}};
+
handle_message(ddoc_updated, _Worker, State) ->
{stop, State}.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 1c4b342..5da5737 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -44,7 +44,7 @@ create(Db, Indexes, Selector, Opts) ->
Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
- Bookmark = couch_util:get_value(bookmark, Opts),
+ Bookmark = couch_util:get_value(bookmark, Opts),
{ok, #cursor{
db = Db,
@@ -124,7 +124,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
BaseArgs = base_args(Cursor),
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
+ Args = mango_json_bookmark:update_args(Bookmark, Args0),
UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
@@ -231,32 +231,26 @@ view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
},
case ViewRow#view_row.doc of
null ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
maybe_send_mango_ping();
undefined ->
- ViewRow2 = ViewRow#view_row{
- value = couch_util:get_value(value, Row)
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
+ % include_docs=false. Use quorum fetch at coordinator
+ ok = rexi:stream2(ViewRow),
set_mango_msg_timestamp();
Doc ->
+ put(mango_docs_examined, get(mango_docs_examined) + 1),
Selector = couch_util:get_value(selector, Options),
case mango_selector:match(Selector, Doc) of
true ->
- ViewRow2 = ViewRow#view_row{
- value = get(mango_docs_examined) + 1
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
+ ok = rexi:stream2(ViewRow),
set_mango_msg_timestamp();
false ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
maybe_send_mango_ping()
end
end,
{ok, Acc};
view_cb(complete, Acc) ->
+ % Send shard-level execution stats
+ ok = rexi:stream2({execution_stats, {docs_examined, get(mango_docs_examined)}}),
% Finish view output
ok = rexi:stream_last(complete),
{ok, Acc};
@@ -286,22 +280,28 @@ handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
case doc_member(Cursor, Props) of
- {ok, Doc, {execution_stats, ExecutionStats1}} ->
+ {ok, Doc, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
- execution_stats = ExecutionStats1
+ execution_stats = Stats
},
Cursor2 = update_bookmark_keys(Cursor1, Props),
FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
handle_doc(Cursor2, FinalDoc);
- {no_match, _, {execution_stats, ExecutionStats1}} ->
+ {no_match, _, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
- execution_stats = ExecutionStats1
+ execution_stats = Stats
},
{ok, Cursor1};
Error ->
couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]),
{ok, Cursor}
end;
+handle_message({execution_stats, ShardStats}, #cursor{execution_stats = Stats} = Cursor) ->
+ {docs_examined, DocsExamined} = ShardStats,
+ Cursor1 = Cursor#cursor{
+ execution_stats = mango_execution_stats:incr_docs_examined(Stats, DocsExamined)
+ },
+ {ok, Cursor1};
handle_message(complete, Cursor) ->
{ok, Cursor};
handle_message({error, Reason}, _Cursor) ->
@@ -410,24 +410,18 @@ apply_opts([{_, _} | Rest], Args) ->
doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
+ Db = Cursor#cursor.db,
Opts = Cursor#cursor.opts,
ExecutionStats = Cursor#cursor.execution_stats,
Selector = Cursor#cursor.selector,
- {Matched, Incr} = case couch_util:get_value(value, RowProps) of
- N when is_integer(N) -> {true, N};
- _ -> {false, 1}
- end,
case couch_util:get_value(doc, RowProps) of
{DocProps} ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats, Incr),
- case Matched of
- true ->
- {ok, {DocProps}, {execution_stats, ExecutionStats1}};
- false ->
- match_doc(Selector, {DocProps}, ExecutionStats1)
- end;
+ % only matching documents are returned; the selector
+ % is evaluated at the shard level in view_cb({row, Row},
+ {ok, {DocProps}, {execution_stats, ExecutionStats}};
undefined ->
+ % an undefined doc was returned, indicating we should
+ % perform a quorum fetch
ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
Id = couch_util:get_value(id, RowProps),
case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
@@ -437,9 +431,9 @@ doc_member(Cursor, RowProps) ->
Else ->
Else
end;
- null ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats),
- {no_match, null, {execution_stats, ExecutionStats1}}
+ _ ->
+ % no doc, no match
+ {no_match, null, {execution_stats, ExecutionStats}}
end.
@@ -460,8 +454,8 @@ is_design_doc(RowProps) ->
update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
+ Id = couch_util:get_value(id, Props),
+ Key = couch_util:get_value(key, Props),
Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key
@@ -506,7 +500,6 @@ does_not_run_match_on_doc_with_value_test() ->
RowProps = [
{id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
{key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {value,1},
{doc,{
[
{<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 922cadf..d3687f8 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -53,6 +53,10 @@ class ExecutionStatsTests(mango.UserDocsTests):
)
self.assertEqual(resp["execution_stats"]["results_returned"], len(resp["docs"]))
+ def test_no_matches_index_scan(self):
+ resp = self.db.find({"age": {"$lt": 35}, "nomatch": "me"}, return_raw=True, executionStats=True)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 0)
@unittest.skipUnless(mango.has_text_service(), "requires text service")
class ExecutionStatsTests_Text(mango.UserDocsTextTests):