You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/13 14:57:59 UTC
[couchdb] 15/20: getting tests to pass
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f48a6374335f8f6327a64a47f43e2677df432a0e
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Feb 10 18:15:00 2020 +0200
getting tests to pass
---
src/mango/src/mango_cursor.erl | 7 ++++++
src/mango/src/mango_execution_stats.erl | 8 ------
src/mango/src/mango_execution_stats.hrl | 1 -
src/mango/src/mango_fdb.erl | 2 +-
src/mango/src/mango_idx.erl | 5 ++--
src/mango/src/mango_idx_special.erl | 4 ++-
src/mango/test/12-use-correct-index-test.py | 38 +++++++++++++++++------------
src/mango/test/15-execution-stats-test.py | 6 ++---
src/mango/test/16-index-selectors-test.py | 2 ++
src/mango/test/17-multi-type-value-test.py | 4 +--
src/mango/test/18-json-sort.py | 6 ++---
src/mango/test/19-find-conflicts.py | 2 +-
src/mango/test/user_docs.py | 4 ++-
13 files changed, 49 insertions(+), 40 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index c6f21dd..8bdf022 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,6 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
+ remove_unbuilt_indexes/1,
maybe_add_warning/3
]).
@@ -123,6 +124,12 @@ remove_indexes_with_partial_filter_selector(Indexes) ->
lists:filter(FiltFun, Indexes).
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, Indexes).
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd7..a3572a1 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -18,7 +18,6 @@
incr_keys_examined/1,
incr_docs_examined/1,
incr_docs_examined/2,
- incr_quorum_docs_examined/1,
incr_results_returned/1,
log_start/1,
log_end/1,
@@ -33,7 +32,6 @@ to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
- {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
]}.
@@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) ->
}.
-incr_quorum_docs_examined(Stats) ->
- Stats#execution_stats {
- totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
- }.
-
-
incr_results_returned(Stats) ->
Stats#execution_stats {
resultsReturned = Stats#execution_stats.resultsReturned + 1
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
index ea5ed5e..783c1e7 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -13,7 +13,6 @@
-record(execution_stats, {
totalKeysExamined = 0,
totalDocsExamined = 0,
- totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
executionTimeMs
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 9a17a85..1274f0e 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -226,7 +226,7 @@ fold_cb({Key, Val}, Acc) ->
} = Acc,
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
SortKeys = couch_views_encoding:decode(Val),
- {ok, Doc} = fabric2_db:open_doc(Db, DocId),
+ {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]),
JSONDoc = couch_doc:to_json_obj(Doc, []),
io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
case Callback({doc, SortKeys, JSONDoc}, Cursor) of
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5be8530..501d8ef 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -103,12 +103,13 @@ get_rev_info(Row) ->
get_usable_indexes(Db, Selector, Opts) ->
- ExistingIndexes = mango_idx:list(Db),
+ ExistingIndexes = mango_idx:add_build_status(Db, mango_idx:list(Db)),
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ GlobalIndexes1 = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(GlobalIndexes1 ++ UserSpecifiedIndex),
UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts),
SortFields = get_sort_fields(Opts),
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..844a0ba 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -27,6 +27,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
-include("mango_idx.hrl").
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?MANGO_INDEX_READY}
]}.
diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py
index 987f507..d495e94 100644
--- a/src/mango/test/12-use-correct-index-test.py
+++ b/src/mango/test/12-use-correct-index-test.py
@@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs(copy.deepcopy(DOCS))
def test_choose_index_with_one_field_in_index(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie"}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
def test_choose_index_with_two(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_alphabetically(self):
- self.db.create_index(["name"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/aaa")
def test_choose_index_most_accurate(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_most_accurate_in_memory_selector(self):
- self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
@@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
def test_chooses_idxA(self):
DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}]
self.db.save_docs(copy.deepcopy(DOCS2))
- self.db.create_index(["a", "b", "c"])
- self.db.create_index(["a", "d", "e"])
+ self.db.create_index(["a", "b", "c"], wait_for_built_index=False)
+ self.db.create_index(["a", "d", "e"], wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find(
{"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True
)
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 922cadf..90430d8 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -37,10 +36,10 @@ class ExecutionStatsTests(mango.UserDocsTests):
resp = self.db.find(
{"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True
)
+ print(resp)
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -63,7 +62,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
self.assertEqual(len(resp["docs"]), 1)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
diff --git a/src/mango/test/16-index-selectors-test.py b/src/mango/test/16-index-selectors-test.py
index 4510065..a3014e9 100644
--- a/src/mango/test/16-index-selectors-test.py
+++ b/src/mango/test/16-index-selectors-test.py
@@ -158,6 +158,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_with_no_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolnoselectorddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschoolnoselector")
self.assertEqual(resp["index"]["name"], "oldschoolnoselector")
docs = self.db.find(selector, use_index="oldschoolnoselector")
@@ -166,6 +167,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschool")
self.assertEqual(resp["index"]["name"], "oldschool")
docs = self.db.find(selector, use_index="oldschool")
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
index 21e7afd..b9420a3 100644
--- a/src/mango/test/17-multi-type-value-test.py
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -53,9 +53,9 @@ class MultiValueFieldTests:
class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["name"], wait_for_built_index=False)
+ self.db.create_index(["age", "name"], wait_for_built_index=False)
self.db.save_docs(copy.deepcopy(DOCS))
- self.db.create_index(["name"])
- self.db.create_index(["age", "name"])
# @unittest.skipUnless(mango.has_text_service(), "requires text service")
diff --git a/src/mango/test/18-json-sort.py b/src/mango/test/18-json-sort.py
index d4e60a3..62c8e29 100644
--- a/src/mango/test/18-json-sort.py
+++ b/src/mango/test/18-json-sort.py
@@ -15,7 +15,7 @@ import copy
import unittest
DOCS = [
- {"_id": "1", "name": "Jimi", "age": 10, "cars": 1},
+ {"_id": "aa", "name": "Jimi", "age": 10, "cars": 1},
{"_id": "2", "name": "Eddie", "age": 20, "cars": 1},
{"_id": "3", "name": "Jane", "age": 30, "cars": 2},
{"_id": "4", "name": "Mary", "age": 40, "cars": 2},
@@ -33,7 +33,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=["age"], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "fwd")
+ self.assertEqual(explain["args"]["direction"], "fwd")
def test_works_for_all_fields_specified(self):
self.db.create_index(["cars", "age"], name="cars-age")
@@ -52,7 +52,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=[{"age": "desc"}], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "rev")
+ self.assertEqual(explain["args"]["direction"], "rev")
def test_not_work_for_non_constant_field(self):
self.db.create_index(["cars", "age"], name="cars-age")
diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py
index bf865d6..45a1e31 100644
--- a/src/mango/test/19-find-conflicts.py
+++ b/src/mango/test/19-find-conflicts.py
@@ -25,7 +25,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs_with_conflicts(copy.deepcopy(CONFLICT))
def test_retrieve_conflicts(self):
- self.db.create_index(["_conflicts"])
+ self.db.create_index(["_conflicts"], wait_for_built_index=False)
result = self.db.find({"_conflicts": {"$exists": True}}, conflicts=True)
self.assertEqual(
result[0]["_conflicts"][0], "1-23202479633c2b380f79507a776743d5"
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 0a52dee..6793601 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -90,7 +90,9 @@ def add_view_indexes(db, kwargs):
(["ordered"], "ordered"),
]
for (idx, name) in indexes:
- assert db.create_index(idx, name=name, ddoc=name) is True
+ assert db.create_index(idx, name=name, ddoc=name,
+ wait_for_built_index=False) is True
+ db.wait_for_built_indexes()
def add_text_indexes(db, kwargs):