You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wi...@apache.org on 2017/10/16 15:01:50 UTC
[couchdb] branch master updated: Fix maximum key value when using
JSON indexes (#881)
This is an automated email from the ASF dual-hosted git repository.
willholley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/master by this push:
new 641aa56 Fix maximum key value when using JSON indexes (#881)
641aa56 is described below
commit 641aa568d011100aacdee6128da51b5a98fdbd8f
Author: Will Holley <wi...@gmail.com>
AuthorDate: Mon Oct 16 16:01:47 2017 +0100
Fix maximum key value when using JSON indexes (#881)
Mango previously constrained range queries against JSON indexes
(map/reduce views) to startkey=[]&endkey=[{}]. In Mango, JSON
index keys are always compound (i.e. always arrays), but this
restriction resulted in Mango failing to match documents where
the indexed value was an object.
For example, an index with keys:
[1],
[2],
[{"foo": 3}]
would be restricted such that only [1] and [2] were returned
if a range query was issued.
On its own, this behaviour isn't necessarily unintuitive, but
it is different from the behaviour of a non-indexed Mango
query, so the query results would change in the presence of an
index.
Additonally, it prevented operators or selectors which explicitly
depend on a full index scan (such as $exists) from returning a
complete result set.
This commit changes the maximum range boundary from {} to a
value that collates higher than any JSON object, so all
array/compound keys will be included.
Note that this uses an invalid UTF-8 character, so we depend
on the view engine not barfing when this is passed as a
parameter. In addition, we can't represent the value in JSON
so we need to subtitute is when returning a query plan
in the _explain endpoint.
---
src/mango/src/mango_cursor_view.erl | 23 +++++++-
src/mango/src/mango_idx_view.erl | 5 +-
src/mango/src/mango_idx_view.hrl | 13 +++++
src/mango/test/02-basic-find-test.py | 2 +-
src/mango/test/17-multi-type-value-test.py | 90 ++++++++++++++++++++++++++++++
5 files changed, 128 insertions(+), 5 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 59dd522..3fcec07 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -29,7 +29,7 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
-include("mango_cursor.hrl").
-
+-include("mango_idx_view.hrl").
create(Db, Indexes, Selector, Opts) ->
FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -61,18 +61,37 @@ explain(Cursor) ->
BaseArgs = base_args(Cursor),
Args = apply_opts(Opts, BaseArgs),
+
[{mrargs, {[
{include_docs, Args#mrargs.include_docs},
{view_type, Args#mrargs.view_type},
{reduce, Args#mrargs.reduce},
{start_key, Args#mrargs.start_key},
- {end_key, Args#mrargs.end_key},
+ {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
{direction, Args#mrargs.direction},
{stable, Args#mrargs.stable},
{update, Args#mrargs.update}
]}}].
+% replace internal values that cannot
+% be represented as a valid UTF-8 string
+% with a token for JSON serialization
+maybe_replace_max_json([]) ->
+ [];
+
+maybe_replace_max_json(?MAX_STR) ->
+ <<"<MAX>">>;
+
+maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
+ H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+ true -> H
+ end,
+ [H1 | maybe_replace_max_json(T)];
+
+maybe_replace_max_json(EndKey) ->
+ EndKey.
+
base_args(#cursor{index = Idx} = Cursor) ->
#mrargs{
view_type = map,
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 8331683..f1041bb 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,6 +34,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include("mango_idx_view.hrl").
validate_new(#idx{}=Idx, _Db) ->
@@ -163,11 +164,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [{[]}];
+ [?MAX_JSON_OBJ];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
- [{[]}];
+ [?MAX_JSON_OBJ];
false ->
[Key | end_key(Rest)]
end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
new file mode 100644
index 0000000..0d213e5
--- /dev/null
+++ b/src/mango/src/mango_idx_view.hrl
@@ -0,0 +1,13 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 72a4e3f..82554a1 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -319,5 +319,5 @@ class BasicFindTests(mango.UserDocsTests):
assert explain["mrargs"]["update"] == True
assert explain["mrargs"]["reduce"] == False
assert explain["mrargs"]["start_key"] == [0]
- assert explain["mrargs"]["end_key"] == [{}]
+ assert explain["mrargs"]["end_key"] == ["<MAX>"]
assert explain["mrargs"]["include_docs"] == True
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
new file mode 100644
index 0000000..d838447
--- /dev/null
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -0,0 +1,90 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+import mango
+import unittest
+
+DOCS = [
+ {
+ "_id": "1",
+ "name": "Jimi",
+ "age": 10
+ },
+ {
+ "_id": "2",
+ "name": {"forename":"Eddie"},
+ "age": 20
+ },
+ {
+ "_id": "3",
+ "name": None,
+ "age": 30
+ },
+ {
+ "_id": "4",
+ "name": 1,
+ "age": 40
+ },
+ {
+ "_id": "5",
+ "forename": "Sam",
+ "age": 50
+ }
+]
+
+
+class MultiValueFieldTests:
+
+ def test_can_query_with_name(self):
+ docs = self.db.find({"name": {"$exists": True}})
+ self.assertEqual(len(docs), 4)
+ for d in docs:
+ self.assertIn("name", d)
+
+ def test_can_query_with_name_subfield(self):
+ docs = self.db.find({"name.forename": {"$exists": True}})
+ self.assertEqual(len(docs), 1)
+ self.assertEqual(docs[0]["_id"], "2")
+
+ def test_can_query_with_name_range(self):
+ docs = self.db.find({"name": {"$gte": 0}})
+ # expect to include "Jimi", 1 and {"forename":"Eddie"}
+ self.assertEqual(len(docs), 3)
+ for d in docs:
+ self.assertIn("name", d)
+
+ def test_can_query_with_age_and_name_range(self):
+ docs = self.db.find({"age": {"$gte": 0, "$lt": 40}, "name": {"$gte": 0}})
+ # expect to include "Jimi", 1 and {"forename":"Eddie"}
+ self.assertEqual(len(docs), 2)
+ for d in docs:
+ self.assertIn("name", d)
+
+
+
+class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
+ def setUp(self):
+ self.db.recreate()
+ self.db.save_docs(copy.deepcopy(DOCS))
+ self.db.create_index(["name"])
+ self.db.create_index(["age", "name"])
+
+# @unittest.skipUnless(mango.has_text_service(), "requires text service")
+# class MultiValueFieldTextTests(MultiValueFieldDocsNoIndexes, OperatorTests):
+# pass
+
+
+class MultiValueFieldAllDocsTests(mango.DbPerClass, MultiValueFieldTests):
+ def setUp(self):
+ self.db.recreate()
+ self.db.save_docs(copy.deepcopy(DOCS))
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].