You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2018/09/13 12:38:08 UTC
[couchdb] branch partition-fix-mango-tests updated (e7b474e ->
fc087cc)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch partition-fix-mango-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard e7b474e add partition support to mango text and fix failing tests
new fc087cc add partition support to mango text and fix failing tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e7b474e)
\
N -- N -- N refs/heads/partition-fix-mango-tests (fc087cc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/mango/src/mango_idx.erl | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
[couchdb] 01/01: add partition support to mango text and fix
failing tests
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch partition-fix-mango-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fc087cc3a7d910407bfc95452b373ce50404b02d
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Sep 13 12:47:38 2018 +0200
add partition support to mango text and fix failing tests
---
src/mango/src/mango_cursor_text.erl | 17 +++++++++++-
src/mango/src/mango_cursor_view.erl | 4 ++-
src/mango/src/mango_idx.erl | 44 ++++++++++++++++++++-----------
src/mango/src/mango_idx_text.erl | 10 ++++---
src/mango/src/mango_idx_view.erl | 12 +--------
src/mango/test/05-index-selection-test.py | 14 +++++-----
src/mango/test/user_docs.py | 3 ++-
7 files changed, 64 insertions(+), 40 deletions(-)
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 3883bc8..7d78e3c 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -25,6 +25,7 @@
-include_lib("dreyfus/include/dreyfus.hrl").
-include("mango_cursor.hrl").
-include("mango.hrl").
+-include("mango_idx.hrl").
-record(cacc, {
@@ -96,6 +97,20 @@ execute(Cursor, UserFun, UserAcc) ->
sort = sort_query(Opts, Selector),
raw_bookmark = true
},
+
+ DbPartitioned = mem3:is_partitioned(couch_db:name(Db)),
+ Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts, DbPartitioned),
+ QueryArgs1 = case Partitioned of
+ true ->
+ Partition = couch_util:get_value(partition, Opts),
+ QueryArgs#index_query_args{
+ partition = Partition,
+ partitioned = true
+ };
+ false ->
+ QueryArgs
+ end,
+
CAcc = #cacc{
selector = Selector,
dbname = couch_db:name(Db),
@@ -104,7 +119,7 @@ execute(Cursor, UserFun, UserAcc) ->
bookmark = get_bookmark(Opts),
limit = Limit,
skip = Skip,
- query_args = QueryArgs,
+ query_args = QueryArgs1,
user_fun = UserFun,
user_acc = UserAcc,
fields = Cursor#cursor.fields,
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 3eedf99..1623420 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -103,7 +103,9 @@ base_args(#cursor{index = Idx, opts = Opts} = Cursor) ->
end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
include_docs = true
},
- Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts),
+
+ DbPartitioned = mem3:is_partitioned(couch_db:name(Cursor#cursor.db)),
+ Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts, DbPartitioned),
Args2 = couch_mrview_util:set_extra(Args1, partitioned, Partitioned),
Args3 = case couch_util:get_value(partition, Opts) of
<<>> ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index e051218..261e8f0 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -23,6 +23,7 @@
new/2,
validate_new/2,
+ validate_design_opts/1,
add/2,
remove/2,
from_ddoc/2,
@@ -59,7 +60,7 @@ list(Db) ->
get_usable_indexes(Db, Selector, Opts) ->
PQ = is_partitioned_query(Opts),
- ExistingIndexes = filter_indexes_by_partitioned(
+ ExistingIndexes = filter_indexes_by_partitioned(mem3:is_partitioned(db_to_name(Db)),
mango_idx:list(Db), PQ),
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
@@ -76,23 +77,24 @@ get_usable_indexes(Db, Selector, Opts) ->
end.
-filter_indexes_by_partitioned(Indexes, PQ) ->
- filter_indexes_by_partitioned(Indexes, PQ, []).
+filter_indexes_by_partitioned(false, Indexes, _PQ) ->
+ Indexes;
+filter_indexes_by_partitioned(DbPartitioned, Indexes, PQ) ->
+ FilterFun = fun (Idx)->
+ PartitionedIdx = case lists:keyfind(partitioned, 1, Idx#idx.design_opts) of
+ {partitioned, PI} -> PI;
+ false -> DbPartitioned
+ end,
+ filter_index_by_partitioned(Idx#idx.def, PartitionedIdx, PQ)
+ end,
+ lists:filter(FilterFun, Indexes).
-filter_indexes_by_partitioned([], _PQ, Acc) ->
- lists:reverse(Acc);
-filter_indexes_by_partitioned([Idx | Rest], PQ, Acc) ->
- {partitioned, PI} = lists:keyfind(partitioned, 1, Idx#idx.design_opts),
- case {Idx#idx.def, PI, PQ} of
- {all_docs, _, _} ->
- % all_docs works both ways.
- filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]);
- {_, Same, Same} ->
- filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]);
- {_, _, _} ->
- filter_indexes_by_partitioned(Rest, PQ, Acc)
- end.
+filter_index_by_partitioned(all_docs, _PartitionedIdx, _PartitionedQuery) ->
+ true;
+
+filter_index_by_partitioned(_Def, PartitionedIdx, PartitionedQuery) ->
+ PartitionedIdx =:= PartitionedQuery.
is_partitioned_query(Opts) ->
@@ -145,6 +147,16 @@ validate_new(Idx, Db) ->
Mod:validate_new(Idx, Db).
+validate_design_opts(Props) ->
+ case lists:keyfind(<<"options">>, 1, Props) of
+ {<<"options">>, {[{<<"partitioned">>, P}]}}
+ when is_boolean(P) ->
+ [{partitioned, P}];
+ _ ->
+ []
+ end.
+
+
add(DDoc, Idx) ->
Mod = idx_mod(Idx),
{ok, NewDDoc} = Mod:add(DDoc, Idx),
diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl
index 29b4441..906bf4d 100644
--- a/src/mango/src/mango_idx_text.erl
+++ b/src/mango/src/mango_idx_text.erl
@@ -51,7 +51,8 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
Texts2 = lists:keystore(element(1, NewText), 1, Texts1, NewText),
Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>,
{Texts2}}),
- {ok, DDoc#doc{body={Props1}}}.
+ Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Idx#idx.design_opts}}),
+ {ok, DDoc#doc{body={Props2}}}.
remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -75,6 +76,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
from_ddoc({Props}) ->
+ DesignOpts = mango_idx:validate_design_opts(Props),
case lists:keyfind(<<"indexes">>, 1, Props) of
{<<"indexes">>, {Texts}} when is_list(Texts) ->
lists:flatmap(fun({Name, {VProps}}) ->
@@ -85,7 +87,8 @@ from_ddoc({Props}) ->
I = #idx{
type = <<"text">>,
name = Name,
- def = Def
+ def = Def,
+ design_opts = DesignOpts
},
[I]
end
@@ -257,7 +260,8 @@ opts() ->
make_text(Idx) ->
Text= {[
{<<"index">>, Idx#idx.def},
- {<<"analyzer">>, construct_analyzer(Idx#idx.def)}
+ {<<"analyzer">>, construct_analyzer(Idx#idx.def)},
+ {<<"options">>, {Idx#idx.opts}}
]},
{Idx#idx.name, Text}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index ef1ca59..c0567f5 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -79,7 +79,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
from_ddoc({Props}) ->
- DesignOpts = validate_design_opts(Props),
+ DesignOpts = mango_idx:validate_design_opts(Props),
case lists:keyfind(<<"views">>, 1, Props) of
{<<"views">>, {Views}} when is_list(Views) ->
lists:flatmap(fun({Name, {VProps}}) ->
@@ -252,16 +252,6 @@ validate_ddoc(VProps) ->
end.
-validate_design_opts(Props) ->
- case lists:keyfind(<<"options">>, 1, Props) of
- {<<"options">>, {[{<<"partitioned">>, P}]}}
- when is_boolean(P) ->
- [{partitioned, P}];
- _ ->
- []
- end.
-
-
% This function returns a list of indexes that
% can be used to restrict this query. This works by
% searching the selector looking for field names that
diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py
index 2a40fda..0c93410 100644
--- a/src/mango/test/05-index-selection-test.py
+++ b/src/mango/test/05-index-selection-test.py
@@ -40,7 +40,7 @@ class IndexSelectionTests:
def test_with_or(self):
# index on ["company","manager"]
- ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198"
+ ddocid = "_design/company-manager"
resp = self.db.find({
"company": {
@@ -56,7 +56,7 @@ class IndexSelectionTests:
def test_use_most_columns(self):
# ddoc id for the age index
- ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+ ddocid = "_design/age"
resp = self.db.find({
"name.first": "Stephanie",
"name.last": "Something or other",
@@ -81,7 +81,7 @@ class IndexSelectionTests:
def test_invalid_use_index(self):
# ddoc id for the age index
- ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+ ddocid = "_design/age"
r = self.db.find({}, use_index=ddocid, return_raw=True)
self.assertEqual(r["warning"], '{0} was not used because it does not contain a valid index for this query.'.format(ddocid))
@@ -101,7 +101,7 @@ class IndexSelectionTests:
def test_reject_use_index_invalid_fields(self):
# index on ["company","manager"] which should not be valid
- ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198"
+ ddocid = "_design/company-manager"
selector = {
"company": "Pharmex"
}
@@ -114,8 +114,8 @@ class IndexSelectionTests:
def test_reject_use_index_ddoc_and_name_invalid_fields(self):
# index on ["company","manager"] which should not be valid
- ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198"
- name = "a0c425a60cf3c3c09e3c537c9ef20059dcef9198"
+ ddocid = "_design/company-manager"
+ name = "company-manager"
selector = {
"company": "Pharmex"
}
@@ -130,7 +130,7 @@ class IndexSelectionTests:
def test_reject_use_index_sort_order(self):
# index on ["company","manager"] which should not be valid
# and there is no valid fallback (i.e. an index on ["company"])
- ddocid = "_design/a0c425a60cf3c3c09e3c537c9ef20059dcef9198"
+ ddocid = "_design/company-manager"
selector = {
"company": {"$gt": None}
}
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 02ffe9f..7455f0a 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -87,7 +87,8 @@ def add_view_indexes(db, kwargs):
["ordered"]
]
for idx in indexes:
- assert db.create_index(idx) is True
+ name = '-'.join(idx)
+ assert db.create_index(idx, ddoc=name, name=name) is True
def add_text_indexes(db, kwargs):