You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2017/08/30 09:51:08 UTC
[couchdb] branch master updated: basic execution statistics for
_find (#768)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/master by this push:
new af839e1 basic execution statistics for _find (#768)
af839e1 is described below
commit af839e101a25cb8abc98b2571ef5344a47df470a
Author: Will Holley <wi...@gmail.com>
AuthorDate: Wed Aug 30 10:51:06 2017 +0100
basic execution statistics for _find (#768)
Accept an "execution_stats" parameter to _find. If present, return
a new "execution_stats" object in the response which contains
information about the query executed. Currently, this is only
implemented for json/all_docs indexes and contains:
- total keys examined (currently always 0 for json indexes)
- total documents examined (when include_docs=true used)
- total quorum documents examined (when fabric doc lookups used)
---
src/mango/src/mango_cursor.hrl | 4 +
src/mango/src/mango_cursor_text.erl | 39 ++++++----
src/mango/src/mango_cursor_view.erl | 37 +++++----
src/mango/src/mango_execution_stats.erl | 89 ++++++++++++++++++++++
...{mango_cursor.hrl => mango_execution_stats.hrl} | 21 ++---
src/mango/src/mango_httpd.erl | 1 +
src/mango/src/mango_opts.erl | 6 ++
src/mango/test/15-execution-stats-test.py | 58 ++++++++++++++
src/mango/test/mango.py | 4 +-
9 files changed, 217 insertions(+), 42 deletions(-)
diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_cursor.hrl
index 956466c..e204c17 100644
--- a/src/mango/src/mango_cursor.hrl
+++ b/src/mango/src/mango_cursor.hrl
@@ -10,6 +10,9 @@
% License for the specific language governing permissions and limitations under
% the License.
+-include("mango_execution_stats.hrl").
+
+
-record(cursor, {
db,
index,
@@ -21,6 +24,7 @@
fields = undefined,
user_fun,
user_acc,
+ execution_stats = #execution_stats{},
bookmark,
bookmark_docid,
bookmark_key
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 96e365a..ea62cd6 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -38,7 +38,8 @@
skip,
user_fun,
user_acc,
- fields
+ fields,
+ execution_stats
}).
@@ -87,7 +88,8 @@ execute(Cursor, UserFun, UserAcc) ->
limit = Limit,
skip = Skip,
selector = Selector,
- opts = Opts
+ opts = Opts,
+ execution_stats = Stats
} = Cursor,
QueryArgs = #index_query_args{
q = mango_selector_text:convert(Selector),
@@ -105,7 +107,8 @@ execute(Cursor, UserFun, UserAcc) ->
query_args = QueryArgs,
user_fun = UserFun,
user_acc = UserAcc,
- fields = Cursor#cursor.fields
+ fields = Cursor#cursor.fields,
+ execution_stats = mango_execution_stats:log_start(Stats)
},
try
execute(CAcc)
@@ -114,12 +117,14 @@ execute(Cursor, UserFun, UserAcc) ->
#cacc{
bookmark = FinalBM,
user_fun = UserFun,
- user_acc = LastUserAcc
+ user_acc = LastUserAcc,
+ execution_stats = Stats0
} = FinalCAcc,
JsonBM = dreyfus_bookmark:pack(FinalBM),
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
- {ok, FinalUserAcc}
+ FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
+ {ok, FinalUserAcc0}
end.
@@ -165,25 +170,28 @@ handle_hits(CAcc0, [{Sort, Doc} | Rest]) ->
handle_hit(CAcc0, Sort, Doc) ->
#cacc{
limit = Limit,
- skip = Skip
+ skip = Skip,
+ execution_stats = Stats
} = CAcc0,
CAcc1 = update_bookmark(CAcc0, Sort),
- case mango_selector:match(CAcc1#cacc.selector, Doc) of
+ Stats1 = mango_execution_stats:incr_docs_examined(Stats),
+ CAcc2 = CAcc1#cacc{execution_stats = Stats1},
+ case mango_selector:match(CAcc2#cacc.selector, Doc) of
true when Skip > 0 ->
- CAcc1#cacc{skip = Skip - 1};
+ CAcc2#cacc{skip = Skip - 1};
true when Limit == 0 ->
% We hit this case if the user spcified with a
% zero limit. Notice that in this case we need
% to return the bookmark from before this match
throw({stop, CAcc0});
true when Limit == 1 ->
- NewCAcc = apply_user_fun(CAcc1, Doc),
+ NewCAcc = apply_user_fun(CAcc2, Doc),
throw({stop, NewCAcc});
true when Limit > 1 ->
- NewCAcc = apply_user_fun(CAcc1, Doc),
+ NewCAcc = apply_user_fun(CAcc2, Doc),
NewCAcc#cacc{limit = Limit - 1};
false ->
- CAcc1
+ CAcc2
end.
@@ -191,13 +199,15 @@ apply_user_fun(CAcc, Doc) ->
FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields),
#cacc{
user_fun = UserFun,
- user_acc = UserAcc
+ user_acc = UserAcc,
+ execution_stats = Stats
} = CAcc,
+ Stats0 = mango_execution_stats:incr_results_returned(Stats),
case UserFun({row, FinalDoc}, UserAcc) of
{ok, NewUserAcc} ->
- CAcc#cacc{user_acc = NewUserAcc};
+ CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0};
{stop, NewUserAcc} ->
- throw({stop, CAcc#cacc{user_acc = NewUserAcc}})
+ throw({stop, CAcc#cacc{user_acc = NewUserAcc, execution_stats = Stats0}})
end.
@@ -296,6 +306,7 @@ get_json_docs(DbName, Hits) ->
Ids = lists:map(fun(#sortable{item = Item}) ->
couch_util:get_value(<<"_id">>, Item#hit.fields)
end, Hits),
+ % TODO: respect R query parameter (same as json indexes)
{ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids),
lists:map(fun(#sortable{item = Item} = Sort) ->
Id = couch_util:get_value(<<"_id">>, Item#hit.fields),
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 189ec66..2dcf1c7 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -70,10 +70,11 @@ explain(Cursor) ->
end.
-execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
+execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
Cursor = Cursor0#cursor{
user_fun = UserFun,
- user_acc = UserAcc
+ user_acc = UserAcc,
+ execution_stats = mango_execution_stats:log_start(Stats)
},
case Cursor#cursor.ranges of
[empty] ->
@@ -108,7 +109,9 @@ execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
NewBookmark = mango_json_bookmark:create(LastCursor),
Arg = {add_key, bookmark, NewBookmark},
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
- {ok, FinalUserAcc};
+ Stats0 = LastCursor#cursor.execution_stats,
+ FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
+ {ok, FinalUserAcc0};
{error, Reason} ->
{error, Reason}
end
@@ -182,15 +185,21 @@ choose_best_index(_DbName, IndexRanges) ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
- case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts) of
- {ok, Doc} ->
- case mango_selector:match(Cursor#cursor.selector, Doc) of
+ case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts, Cursor#cursor.execution_stats) of
+ {ok, Doc, {execution_stats, ExecutionStats1}} ->
+ Cursor1 = Cursor#cursor {
+ execution_stats = ExecutionStats1
+ },
+ case mango_selector:match(Cursor1#cursor.selector, Doc) of
true ->
- Cursor1 = update_bookmark_keys(Cursor, Props),
- FinalDoc = mango_fields:extract(Doc, Cursor1#cursor.fields),
- handle_doc(Cursor1, FinalDoc);
+ Cursor2 = update_bookmark_keys(Cursor1, Props),
+ FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
+ Cursor3 = Cursor2#cursor {
+ execution_stats = mango_execution_stats:incr_results_returned(Cursor2#cursor.execution_stats)
+ },
+ handle_doc(Cursor3, FinalDoc);
false ->
- {ok, Cursor}
+ {ok, Cursor1}
end;
Error ->
couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]),
@@ -298,15 +307,17 @@ apply_opts([{_, _} | Rest], Args) ->
apply_opts(Rest, Args).
-doc_member(Db, RowProps, Opts) ->
+doc_member(Db, RowProps, Opts, ExecutionStats) ->
case couch_util:get_value(doc, RowProps) of
{DocProps} ->
- {ok, {DocProps}};
+ ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats),
+ {ok, {DocProps}, {execution_stats, ExecutionStats1}};
undefined ->
+ ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
Id = couch_util:get_value(id, RowProps),
case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
{ok, #doc{}=Doc} ->
- {ok, couch_doc:to_json_obj(Doc, [])};
+ {ok, couch_doc:to_json_obj(Doc, []), {execution_stats, ExecutionStats1}};
Else ->
Else
end
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
new file mode 100644
index 0000000..95b9038
--- /dev/null
+++ b/src/mango/src/mango_execution_stats.erl
@@ -0,0 +1,89 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_execution_stats).
+
+
+-export([
+ to_json/1,
+ incr_keys_examined/1,
+ incr_docs_examined/1,
+ incr_quorum_docs_examined/1,
+ incr_results_returned/1,
+ log_start/1,
+ log_end/1,
+ maybe_add_stats/4
+]).
+
+
+-include("mango_cursor.hrl").
+
+
+to_json(Stats) ->
+ {[
+ {total_keys_examined, Stats#execution_stats.totalKeysExamined},
+ {total_docs_examined, Stats#execution_stats.totalDocsExamined},
+ {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
+ {results_returned, Stats#execution_stats.resultsReturned},
+ {execution_time_ms, Stats#execution_stats.executionTimeMs}
+ ]}.
+
+
+incr_keys_examined(Stats) ->
+ Stats#execution_stats {
+ totalKeysExamined = Stats#execution_stats.totalKeysExamined + 1
+ }.
+
+
+incr_docs_examined(Stats) ->
+ Stats#execution_stats {
+ totalDocsExamined = Stats#execution_stats.totalDocsExamined + 1
+ }.
+
+
+incr_quorum_docs_examined(Stats) ->
+ Stats#execution_stats {
+ totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
+ }.
+
+
+incr_results_returned(Stats) ->
+ Stats#execution_stats {
+ resultsReturned = Stats#execution_stats.resultsReturned + 1
+ }.
+
+
+log_start(Stats) ->
+ Stats#execution_stats {
+ executionStartTime = now()
+ }.
+
+
+log_end(Stats) ->
+ End = now(),
+ Diff = timer:now_diff(End, Stats#execution_stats.executionStartTime) / 1000,
+ Stats#execution_stats {
+ executionTimeMs = Diff
+ }.
+
+
+maybe_add_stats(Opts, UserFun, Stats, UserAcc) ->
+ case couch_util:get_value(execution_stats, Opts) of
+ true ->
+ Stats0 = log_end(Stats),
+ JSONValue = to_json(Stats0),
+ Arg = {add_key, execution_stats, JSONValue},
+ {_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
+ FinalUserAcc;
+ _ ->
+ UserAcc
+ end.
\ No newline at end of file
diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_execution_stats.hrl
similarity index 73%
copy from src/mango/src/mango_cursor.hrl
copy to src/mango/src/mango_execution_stats.hrl
index 956466c..ea5ed5e 100644
--- a/src/mango/src/mango_cursor.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -10,18 +10,11 @@
% License for the specific language governing permissions and limitations under
% the License.
--record(cursor, {
- db,
- index,
- ranges,
- selector,
- opts,
- limit,
- skip = 0,
- fields = undefined,
- user_fun,
- user_acc,
- bookmark,
- bookmark_docid,
- bookmark_key
+-record(execution_stats, {
+ totalKeysExamined = 0,
+ totalDocsExamined = 0,
+ totalQuorumDocsExamined = 0,
+ resultsReturned = 0,
+ executionStartTime,
+ executionTimeMs
}).
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 5bc61f5..a99b054 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -21,6 +21,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include("mango_execution_stats.hrl").
-record(vacc, {
resp,
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index fe28d8d..7bae9c9 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -146,6 +146,12 @@ validate_find({Props}) ->
{optional, true},
{default, false},
{validator, fun mango_opts:is_boolean/1}
+ ]},
+ {<<"execution_stats">>, [
+ {tag, execution_stats},
+ {optional, true},
+ {default, false},
+ {validator, fun mango_opts:is_boolean/1}
]}
],
validate(Props, Opts).
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
new file mode 100644
index 0000000..67c9e64
--- /dev/null
+++ b/src/mango/test/15-execution-stats-test.py
@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+
+import mango
+import unittest
+
+class ExecutionStatsTests(mango.UserDocsTests):
+
+ def test_simple_json_index(self):
+ resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=True)
+ self.assertEqual(len(resp["docs"]), 3)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 3)
+ self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+ def test_no_execution_stats(self):
+ resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, executionStats=False)
+ assert "execution_stats" not in resp
+
+ def test_quorum_json_index(self):
+ resp = self.db.find({"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True)
+ self.assertEqual(len(resp["docs"]), 3)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 3)
+ self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class ExecutionStatsTests_Text(mango.UserDocsTextTests):
+
+ def test_simple_text_index(self):
+ resp = self.db.find({"$text": "Stephanie"},
+ return_raw=True,
+ executionStats=True)
+ self.assertEqual(len(resp["docs"]), 1)
+ self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
+ self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
+ self.assertEqual(resp["execution_stats"]["results_returned"], 1)
+ self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
+
+ def test_no_execution_stats(self):
+ resp = self.db.find({"$text": "Stephanie"},
+ return_raw=True)
+ self.assertNotIn("execution_stats", resp)
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index efcf67f..dbe980e 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -155,7 +155,7 @@ class Database(object):
def find(self, selector, limit=25, skip=0, sort=None, fields=None,
r=1, conflicts=False, use_index=None, explain=False,
- bookmark=None, return_raw=False, update=True):
+ bookmark=None, return_raw=False, update=True, executionStats=False):
body = {
"selector": selector,
"use_index": use_index,
@@ -172,6 +172,8 @@ class Database(object):
body["bookmark"] = bookmark
if update == False:
body["update"] = False
+ if executionStats == True:
+ body["execution_stats"] = True
body = json.dumps(body)
if explain:
path = self.path("_explain")
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].