You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2017/09/14 12:28:44 UTC
[couchdb] branch master updated: Add selector support for json
indexes (#808)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/master by this push:
new ed6ec66 Add selector support for json indexes (#808)
ed6ec66 is described below
commit ed6ec66972f268b6d6d86b55730e73cc4ab54ceb
Author: garren smith <ga...@gmail.com>
AuthorDate: Thu Sep 14 14:28:42 2017 +0200
Add selector support for json indexes (#808)
* Add selector support for json indexes
Adds selector support to json indexes. The selector can be used to
filter what documents are added to the index. When executing a query
the index will only be used if the index is specified in the use_index
field.
---
src/mango/src/mango_cursor.erl | 18 +++-
src/mango/src/mango_idx.erl | 15 +++-
src/mango/src/mango_idx_view.erl | 6 ++
src/mango/src/mango_native_proc.erl | 38 ++++++---
src/mango/test/16-index-selectors.py | 156 +++++++++++++++++++++++++++++++++++
src/mango/test/mango.py | 6 +-
6 files changed, 219 insertions(+), 20 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 12ce2c9..f36febd 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -17,7 +17,7 @@
create/3,
explain/1,
execute/3,
- maybe_filter_indexes/2,
+ maybe_filter_indexes_by_ddoc/2,
maybe_add_warning/3
]).
@@ -87,10 +87,12 @@ execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
Mod:execute(Cursor, UserFun, UserAcc).
-maybe_filter_indexes(Indexes, Opts) ->
+maybe_filter_indexes_by_ddoc(Indexes, Opts) ->
case lists:keyfind(use_index, 1, Opts) of
{use_index, []} ->
- Indexes;
+ %We remove any indexes that have a selector
+ % since they are only used when specified via use_index
+ remove_indexes_with_selector(Indexes);
{use_index, [DesignId]} ->
filter_indexes(Indexes, DesignId);
{use_index, [DesignId, ViewName]} ->
@@ -115,6 +117,16 @@ filter_indexes(Indexes0, DesignId, ViewName) ->
lists:filter(FiltFun, Indexes).
+remove_indexes_with_selector(Indexes) ->
+ FiltFun = fun(Idx) ->
+ case mango_idx:get_idx_selector(Idx) of
+ undefined -> true;
+ _ -> false
+ end
+ end,
+ lists:filter(FiltFun, Indexes).
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index c330702..b812251 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -43,7 +43,8 @@
idx_mod/1,
to_json/1,
delete/4,
- get_usable_indexes/3
+ get_usable_indexes/3,
+ get_idx_selector/1
]).
@@ -64,7 +65,7 @@ get_usable_indexes(Db, Selector0, Opts) ->
?MANGO_ERROR({no_usable_index, no_indexes_defined})
end,
- FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts),
+ FilteredIndexes = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
if FilteredIndexes /= [] -> ok; true ->
?MANGO_ERROR({no_usable_index, no_index_matching_name})
end,
@@ -367,3 +368,13 @@ filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
+get_idx_selector(#idx{def = Def}) when Def =:= all_docs; Def =:= undefined ->
+ undefined;
+get_idx_selector(#idx{def = {Def}}) ->
+ case proplists:get_value(<<"selector">>, Def) of
+ undefined -> undefined;
+ {[]} -> undefined;
+ Selector -> Selector
+ end.
+
+
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 8bad34c..d5dcd0c 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -197,6 +197,12 @@ opts() ->
{<<"fields">>, [
{tag, fields},
{validator, fun mango_opts:validate_sort/1}
+ ]},
+ {<<"selector">>, [
+ {tag, selector},
+ {optional, true},
+ {default, {[]}},
+ {validator, fun mango_opts:validate_selector/1}
]}
].
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
index ba17c48..82081a9 100644
--- a/src/mango/src/mango_native_proc.erl
+++ b/src/mango/src/mango_native_proc.erl
@@ -135,26 +135,31 @@ index_doc(#st{indexes=Indexes}, Doc) ->
get_index_entries({IdxProps}, Doc) ->
{Fields} = couch_util:get_value(<<"fields">>, IdxProps),
- Values = lists:map(fun({Field, _Dir}) ->
+ Selector = get_index_selector(IdxProps),
+ case should_index(Selector, Doc) of
+ false ->
+ [];
+ true ->
+ Values = get_index_values(Fields, Doc),
+ case lists:member(not_found, Values) of
+ true -> [];
+ false -> [[Values, null]]
+ end
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
case mango_doc:get_field(Doc, Field) of
not_found -> not_found;
bad_path -> not_found;
- Else -> Else
+ Value -> Value
end
- end, Fields),
- case lists:member(not_found, Values) of
- true ->
- [];
- false ->
- [[Values, null]]
- end.
+ end, Fields).
get_text_entries({IdxProps}, Doc) ->
- Selector = case couch_util:get_value(<<"selector">>, IdxProps) of
- [] -> {[]};
- Else -> Else
- end,
+ Selector = get_index_selector(IdxProps),
case should_index(Selector, Doc) of
true ->
get_text_entries0(IdxProps, Doc);
@@ -163,6 +168,13 @@ get_text_entries({IdxProps}, Doc) ->
end.
+get_index_selector(IdxProps) ->
+ case couch_util:get_value(<<"selector">>, IdxProps) of
+ [] -> {[]};
+ Else -> Else
+ end.
+
+
get_text_entries0(IdxProps, Doc) ->
DefaultEnabled = get_default_enabled(IdxProps),
IndexArrayLengths = get_index_array_lengths(IdxProps),
diff --git a/src/mango/test/16-index-selectors.py b/src/mango/test/16-index-selectors.py
new file mode 100644
index 0000000..b189456
--- /dev/null
+++ b/src/mango/test/16-index-selectors.py
@@ -0,0 +1,156 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+import mango
+import unittest
+
+DOCS = [
+ {
+ "_id": "100",
+ "name": "Jimi",
+ "location": "AUS",
+ "user_id": 1,
+ "same": "value"
+ },
+ {
+ "_id": "200",
+ "name": "Eddie",
+ "location": "BRA",
+ "user_id": 2,
+ "same": "value"
+ },
+ {
+ "_id": "300",
+ "name": "Harry",
+ "location": "CAN",
+ "user_id":3,
+ "same": "value"
+ },
+ {
+ "_id": "400",
+ "name": "Eddie",
+ "location": "DEN",
+ "user_id":4,
+ "same": "value"
+ },
+ {
+ "_id": "500",
+ "name": "Jones",
+ "location": "ETH",
+ "user_id":5,
+ "same": "value"
+ },
+ {
+ "_id": "600",
+ "name": "Winnifried",
+ "location": "FRA",
+ "user_id":6,
+ "same": "value"
+ },
+ {
+ "_id": "700",
+ "name": "Marilyn",
+ "location": "GHA",
+ "user_id":7,
+ "same": "value"
+ },
+ {
+ "_id": "800",
+ "name": "Sandra",
+ "location": "ZAR",
+ "user_id":8,
+ "same": "value"
+ },
+]
+
+class IndexSelectorJson(mango.DbPerClass):
+ def setUp(self):
+ self.db.recreate()
+ self.db.save_docs(copy.deepcopy(DOCS))
+
+ def test_saves_selector_in_index(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_index(["location"], selector=selector)
+ indexes = self.db.list_indexes()
+ self.assertEqual(indexes[1]["def"]["selector"], selector)
+
+ def test_uses_partial_index_for_query_selector(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector, explain=True, use_index='Selected')
+ self.assertEqual(resp["index"]["name"], "Selected")
+ docs = self.db.find(selector, use_index='Selected')
+ self.assertEqual(len(docs), 3)
+
+ def test_uses_partial_index_with_different_selector(self):
+ selector = {"location": {"$gte": "FRA"}}
+ selector2 = {"location": {"$gte": "A"}}
+ self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector2, explain=True, use_index='Selected')
+ self.assertEqual(resp["index"]["name"], "Selected")
+ docs = self.db.find(selector2, use_index='Selected')
+ self.assertEqual(len(docs), 3)
+
+ def test_doesnot_use_selector_when_not_specified(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector, explain=True)
+ self.assertEqual(resp["index"]["name"], "_all_docs")
+
+ def test_doesnot_use_selector_when_not_specified_with_index(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+ self.db.create_index(["location"], name="NotSelected")
+ resp = self.db.find(selector, explain=True)
+ self.assertEqual(resp["index"]["name"], "NotSelected")
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_text_saves_selector_in_index(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector)
+ indexes = self.db.list_indexes()
+ self.assertEqual(indexes[1]["def"]["selector"], selector)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_text_uses_partial_index_for_query_selector(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector, explain=True, use_index='Selected')
+ self.assertEqual(resp["index"]["name"], "Selected")
+ docs = self.db.find(selector, use_index='Selected', fields=['_id', 'location'])
+ self.assertEqual(len(docs), 3)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_text_uses_partial_index_with_different_selector(self):
+ selector = {"location": {"$gte": "FRA"}}
+ selector2 = {"location": {"$gte": "A"}}
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector2, explain=True, use_index='Selected')
+ self.assertEqual(resp["index"]["name"], "Selected")
+ docs = self.db.find(selector2, use_index='Selected')
+ self.assertEqual(len(docs), 3)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_text_doesnot_use_selector_when_not_specified(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+ resp = self.db.find(selector, explain=True)
+ self.assertEqual(resp["index"]["name"], "_all_docs")
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_text_doesnot_use_selector_when_not_specified_with_index(self):
+ selector = {"location": {"$gte": "FRA"}}
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+ self.db.create_text_index(fields=[{"name":"location", "type":"string"}], name="NotSelected")
+ resp = self.db.find(selector, explain=True)
+ self.assertEqual(resp["index"]["name"], "NotSelected")
\ No newline at end of file
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 69b3649..2c89714 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -84,7 +84,7 @@ class Database(object):
r.raise_for_status()
return r.json()
- def create_index(self, fields, idx_type="json", name=None, ddoc=None):
+ def create_index(self, fields, idx_type="json", name=None, ddoc=None, selector=None):
body = {
"index": {
"fields": fields
@@ -96,6 +96,8 @@ class Database(object):
body["name"] = name
if ddoc is not None:
body["ddoc"] = ddoc
+ if selector is not None:
+ body["index"]["selector"] = selector
body = json.dumps(body)
r = self.sess.post(self.path("_index"), data=body)
r.raise_for_status()
@@ -120,7 +122,7 @@ class Database(object):
if index_array_lengths is not None:
body["index"]["index_array_lengths"] = index_array_lengths
if selector is not None:
- body["selector"] = selector
+ body["index"]["selector"] = selector
if fields is not None:
body["index"]["fields"] = fields
if ddoc is not None:
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].