You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2017/09/12 12:06:43 UTC
[couchdb] branch master updated (aa14e5d -> bc43efb)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from aa14e5d Query operator tests for multiple index types (#800)
new 04e4cfa Avoid duplicate index selection in Mango
new bc43efb Use unittest assert in index selection tests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/mango/src/mango_cursor.erl | 15 ++++++++++++++-
src/mango/src/mango_cursor_text.erl | 3 ++-
src/mango/src/mango_cursor_view.erl | 3 ++-
src/mango/src/mango_httpd.erl | 16 +++-------------
src/mango/test/12-use-correct-index.py | 19 +++++++++++++------
5 files changed, 34 insertions(+), 22 deletions(-)
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].
[couchdb] 02/02: Use unittest assert in index selection tests
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bc43efbbe2bca4af98b621dd597d3546b00bfb4e
Author: Will Holley <wi...@gmail.com>
AuthorDate: Tue Sep 12 11:23:44 2017 +0100
Use unittest assert in index selection tests
Replace use of native assert with unittest.assertX.
This ensures we return descriptive errors when assertions
fail.
---
src/mango/test/12-use-correct-index.py | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/src/mango/test/12-use-correct-index.py b/src/mango/test/12-use-correct-index.py
index aaa7980..84b4253 100644
--- a/src/mango/test/12-use-correct-index.py
+++ b/src/mango/test/12-use-correct-index.py
@@ -58,42 +58,42 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
self.db.create_index(["name"], ddoc="zzz")
explain = self.db.find({"name": "Eddie"}, explain=True)
- assert explain["index"]["ddoc"] == '_design/zzz'
+ self.assertEqual(explain["index"]["ddoc"], '_design/zzz')
def test_choose_index_with_two(self):
self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
self.db.create_index(["name", "age"], ddoc="bbb")
self.db.create_index(["name"], ddoc="zzz")
explain = self.db.find({"name": "Eddie", "age":{"$gte": 12}}, explain=True)
- assert explain["index"]["ddoc"] == '_design/bbb'
+ self.assertEqual(explain["index"]["ddoc"], '_design/bbb')
def test_choose_index_alphabetically(self):
self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
self.db.create_index(["name", "age", "location"], ddoc="bbb")
self.db.create_index(["name"], ddoc="zzz")
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
- assert explain["index"]["ddoc"] == '_design/aaa'
+ self.assertEqual(explain["index"]["ddoc"], '_design/aaa')
def test_choose_index_most_accurate(self):
self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
self.db.create_index(["name"], ddoc="zzz")
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
- assert explain["index"]["ddoc"] == '_design/bbb'
+ self.assertEqual(explain["index"]["ddoc"], '_design/bbb')
def test_choose_index_most_accurate_in_memory_selector(self):
self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
self.db.create_index(["name"], ddoc="zzz")
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
- assert explain["index"]["ddoc"] == '_design/zzz'
+ self.assertEqual(explain["index"]["ddoc"], '_design/zzz')
def test_warn_on_full_db_scan(self):
selector = {"not_indexed":"foo"}
explain_resp = self.db.find(selector, explain=True, return_raw=True)
- assert explain_resp["index"]["type"] == "special"
+ self.assertEqual(explain_resp["index"]["type"], "special")
resp = self.db.find(selector, return_raw=True)
- assert resp["warning"] == "no matching index found, create an index to optimize query time"
+ self.assertEqual(resp["warning"], "no matching index found, create an index to optimize query time")
def test_chooses_idxA(self):
DOCS2 = [
@@ -104,4 +104,4 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.create_index(["a", "b", "c"])
self.db.create_index(["a", "d", "e"])
explain = self.db.find({"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True)
- assert explain["index"]["def"]["fields"] == [{'a': 'asc'}, {'b': 'asc'}, {'c': 'asc'}]
+ self.assertEqual(explain["index"]["def"]["fields"], [{'a': 'asc'}, {'b': 'asc'}, {'c': 'asc'}])
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.
[couchdb] 01/02: Avoid duplicate index selection in Mango
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 04e4cfa0c670c89324887d9c1c28139cca65a230
Author: Will Holley <wi...@gmail.com>
AuthorDate: Tue Sep 12 07:55:44 2017 +0100
Avoid duplicate index selection in Mango
Previously, index selection for a given query
was run twice for each request - once to add
a warning in case a full database scan would be
performed and then again when the query was executed.
This moves the warning generation so that it occurs
at the end of the query processing and we can use
the existing index context to decide whether to
add a warning or not.
Whilst only a minor optimisation (which also assumes
we don't have cached query plans etc), it
at least moves index selection to where you'd expect
it to happen (query planning).
---
src/mango/src/mango_cursor.erl | 15 ++++++++++++++-
src/mango/src/mango_cursor_text.erl | 3 ++-
src/mango/src/mango_cursor_view.erl | 3 ++-
src/mango/src/mango_httpd.erl | 16 +++-------------
src/mango/test/12-use-correct-index.py | 7 +++++++
5 files changed, 28 insertions(+), 16 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index cf71790..12ce2c9 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -17,13 +17,15 @@
create/3,
explain/1,
execute/3,
- maybe_filter_indexes/2
+ maybe_filter_indexes/2,
+ maybe_add_warning/3
]).
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_cursor.hrl").
+-include("mango_idx.hrl").
-ifdef(HAVE_DREYFUS).
@@ -134,3 +136,14 @@ group_indexes_by_type(Indexes) ->
[]
end
end, ?CURSOR_MODULES).
+
+
+maybe_add_warning(UserFun, #idx{type = IndexType}, UserAcc) ->
+ case IndexType of
+ <<"special">> ->
+ Arg = {add_key, warning, <<"no matching index found, create an index to optimize query time">>},
+ {_Go, UserAcc0} = UserFun(Arg, UserAcc),
+ UserAcc0;
+ _ ->
+ UserAcc
+ end.
\ No newline at end of file
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index ea62cd6..70c911a 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -124,7 +124,8 @@ execute(Cursor, UserFun, UserAcc) ->
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- {ok, FinalUserAcc0}
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Idx, FinalUserAcc0),
+ {ok, FinalUserAcc1}
end.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index b292704..31e198f 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -118,7 +118,8 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
Stats0 = LastCursor#cursor.execution_stats,
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- {ok, FinalUserAcc0};
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Idx, FinalUserAcc0),
+ {ok, FinalUserAcc1};
{error, Reason} ->
{error, Reason}
end
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index a99b054..33e0c1c 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -183,7 +183,7 @@ handle_find_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
- {ok, Resp0} = start_find_resp(Req, Db, Sel, Opts),
+ {ok, Resp0} = start_find_resp(Req),
{ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
end_find_resp(AccOut);
@@ -230,18 +230,8 @@ convert_to_design_id(DDocId) ->
end.
-start_find_resp(Req, Db, Sel, Opts) ->
- chttpd:start_delayed_json_response(Req, 200, [], maybe_add_warning(Db, Sel, Opts)).
-
-
-maybe_add_warning(Db, Selector, Opts) ->
- UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
- case length(UsableIndexes) of
- 0 ->
- "{\"warning\":\"no matching index found, create an index to optimize query time\",\r\n\"docs\":[";
- _ ->
- "{\"docs\":["
- end.
+start_find_resp(Req) ->
+ chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":[").
end_find_resp(Acc0) ->
diff --git a/src/mango/test/12-use-correct-index.py b/src/mango/test/12-use-correct-index.py
index f1eaf5f..aaa7980 100644
--- a/src/mango/test/12-use-correct-index.py
+++ b/src/mango/test/12-use-correct-index.py
@@ -88,6 +88,13 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
assert explain["index"]["ddoc"] == '_design/zzz'
+ def test_warn_on_full_db_scan(self):
+ selector = {"not_indexed":"foo"}
+ explain_resp = self.db.find(selector, explain=True, return_raw=True)
+ assert explain_resp["index"]["type"] == "special"
+ resp = self.db.find(selector, return_raw=True)
+ assert resp["warning"] == "no matching index found, create an index to optimize query time"
+
def test_chooses_idxA(self):
DOCS2 = [
{"a":1, "b":1, "c":1},
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.