You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/10 16:15:16 UTC
[couchdb] branch fdb-mango-indexes updated (324f83b -> d8312c9)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 324f83b Wrap lines to 80 chars and remove trailing whitespace
new 5d753b6 add bookmark support
new d8312c9 getting tests to pass
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/mango/src/mango_cursor.erl | 7 +++++
src/mango/src/mango_cursor_view.erl | 48 +++++++++++++++++++----------
src/mango/src/mango_execution_stats.erl | 8 -----
src/mango/src/mango_execution_stats.hrl | 1 -
src/mango/src/mango_fdb.erl | 47 +++++++++++++++-------------
src/mango/src/mango_idx.erl | 6 ++--
src/mango/src/mango_idx_special.erl | 4 ++-
src/mango/test/12-use-correct-index-test.py | 38 +++++++++++++----------
src/mango/test/15-execution-stats-test.py | 6 ++--
src/mango/test/16-index-selectors-test.py | 2 ++
src/mango/test/17-multi-type-value-test.py | 4 +--
src/mango/test/18-json-sort.py | 6 ++--
src/mango/test/19-find-conflicts.py | 2 +-
src/mango/test/mango.py | 15 +++++++--
src/mango/test/user_docs.py | 4 ++-
15 files changed, 119 insertions(+), 79 deletions(-)
[couchdb] 02/02: getting tests to pass
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d8312c96322e1e58ec1474b82aed0ab0dc6d9ede
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Feb 10 18:15:00 2020 +0200
getting tests to pass
---
src/mango/src/mango_cursor.erl | 7 ++++++
src/mango/src/mango_execution_stats.erl | 8 ------
src/mango/src/mango_execution_stats.hrl | 1 -
src/mango/src/mango_fdb.erl | 2 +-
src/mango/src/mango_idx.erl | 5 ++--
src/mango/src/mango_idx_special.erl | 4 ++-
src/mango/test/12-use-correct-index-test.py | 38 +++++++++++++++++------------
src/mango/test/15-execution-stats-test.py | 6 ++---
src/mango/test/16-index-selectors-test.py | 2 ++
src/mango/test/17-multi-type-value-test.py | 4 +--
src/mango/test/18-json-sort.py | 6 ++---
src/mango/test/19-find-conflicts.py | 2 +-
src/mango/test/user_docs.py | 4 ++-
13 files changed, 49 insertions(+), 40 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index c6f21dd..8bdf022 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,6 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
+ remove_unbuilt_indexes/1,
maybe_add_warning/3
]).
@@ -123,6 +124,12 @@ remove_indexes_with_partial_filter_selector(Indexes) ->
lists:filter(FiltFun, Indexes).
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, Indexes).
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd7..a3572a1 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -18,7 +18,6 @@
incr_keys_examined/1,
incr_docs_examined/1,
incr_docs_examined/2,
- incr_quorum_docs_examined/1,
incr_results_returned/1,
log_start/1,
log_end/1,
@@ -33,7 +32,6 @@ to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
- {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
]}.
@@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) ->
}.
-incr_quorum_docs_examined(Stats) ->
- Stats#execution_stats {
- totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
- }.
-
-
incr_results_returned(Stats) ->
Stats#execution_stats {
resultsReturned = Stats#execution_stats.resultsReturned + 1
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
index ea5ed5e..783c1e7 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -13,7 +13,6 @@
-record(execution_stats, {
totalKeysExamined = 0,
totalDocsExamined = 0,
- totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
executionTimeMs
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 9a17a85..1274f0e 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -226,7 +226,7 @@ fold_cb({Key, Val}, Acc) ->
} = Acc,
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
SortKeys = couch_views_encoding:decode(Val),
- {ok, Doc} = fabric2_db:open_doc(Db, DocId),
+ {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]),
JSONDoc = couch_doc:to_json_obj(Doc, []),
io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
case Callback({doc, SortKeys, JSONDoc}, Cursor) of
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5be8530..501d8ef 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -103,12 +103,13 @@ get_rev_info(Row) ->
get_usable_indexes(Db, Selector, Opts) ->
- ExistingIndexes = mango_idx:list(Db),
+ ExistingIndexes = mango_idx:add_build_status(Db, mango_idx:list(Db)),
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ GlobalIndexes1 = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(GlobalIndexes1 ++ UserSpecifiedIndex),
UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts),
SortFields = get_sort_fields(Opts),
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..844a0ba 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -27,6 +27,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
-include("mango_idx.hrl").
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?MANGO_INDEX_READY}
]}.
diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py
index 987f507..d495e94 100644
--- a/src/mango/test/12-use-correct-index-test.py
+++ b/src/mango/test/12-use-correct-index-test.py
@@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs(copy.deepcopy(DOCS))
def test_choose_index_with_one_field_in_index(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie"}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
def test_choose_index_with_two(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_alphabetically(self):
- self.db.create_index(["name"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/aaa")
def test_choose_index_most_accurate(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_most_accurate_in_memory_selector(self):
- self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
@@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
def test_chooses_idxA(self):
DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}]
self.db.save_docs(copy.deepcopy(DOCS2))
- self.db.create_index(["a", "b", "c"])
- self.db.create_index(["a", "d", "e"])
+ self.db.create_index(["a", "b", "c"], wait_for_built_index=False)
+ self.db.create_index(["a", "d", "e"], wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find(
{"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True
)
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 922cadf..90430d8 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -37,10 +36,10 @@ class ExecutionStatsTests(mango.UserDocsTests):
resp = self.db.find(
{"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True
)
+ print(resp)
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -63,7 +62,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
self.assertEqual(len(resp["docs"]), 1)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
diff --git a/src/mango/test/16-index-selectors-test.py b/src/mango/test/16-index-selectors-test.py
index 4510065..a3014e9 100644
--- a/src/mango/test/16-index-selectors-test.py
+++ b/src/mango/test/16-index-selectors-test.py
@@ -158,6 +158,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_with_no_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolnoselectorddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschoolnoselector")
self.assertEqual(resp["index"]["name"], "oldschoolnoselector")
docs = self.db.find(selector, use_index="oldschoolnoselector")
@@ -166,6 +167,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschool")
self.assertEqual(resp["index"]["name"], "oldschool")
docs = self.db.find(selector, use_index="oldschool")
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
index 21e7afd..b9420a3 100644
--- a/src/mango/test/17-multi-type-value-test.py
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -53,9 +53,9 @@ class MultiValueFieldTests:
class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["name"], wait_for_built_index=False)
+ self.db.create_index(["age", "name"], wait_for_built_index=False)
self.db.save_docs(copy.deepcopy(DOCS))
- self.db.create_index(["name"])
- self.db.create_index(["age", "name"])
# @unittest.skipUnless(mango.has_text_service(), "requires text service")
diff --git a/src/mango/test/18-json-sort.py b/src/mango/test/18-json-sort.py
index d4e60a3..62c8e29 100644
--- a/src/mango/test/18-json-sort.py
+++ b/src/mango/test/18-json-sort.py
@@ -15,7 +15,7 @@ import copy
import unittest
DOCS = [
- {"_id": "1", "name": "Jimi", "age": 10, "cars": 1},
+ {"_id": "aa", "name": "Jimi", "age": 10, "cars": 1},
{"_id": "2", "name": "Eddie", "age": 20, "cars": 1},
{"_id": "3", "name": "Jane", "age": 30, "cars": 2},
{"_id": "4", "name": "Mary", "age": 40, "cars": 2},
@@ -33,7 +33,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=["age"], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "fwd")
+ self.assertEqual(explain["args"]["direction"], "fwd")
def test_works_for_all_fields_specified(self):
self.db.create_index(["cars", "age"], name="cars-age")
@@ -52,7 +52,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=[{"age": "desc"}], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "rev")
+ self.assertEqual(explain["args"]["direction"], "rev")
def test_not_work_for_non_constant_field(self):
self.db.create_index(["cars", "age"], name="cars-age")
diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py
index bf865d6..45a1e31 100644
--- a/src/mango/test/19-find-conflicts.py
+++ b/src/mango/test/19-find-conflicts.py
@@ -25,7 +25,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs_with_conflicts(copy.deepcopy(CONFLICT))
def test_retrieve_conflicts(self):
- self.db.create_index(["_conflicts"])
+ self.db.create_index(["_conflicts"], wait_for_built_index=False)
result = self.db.find({"_conflicts": {"$exists": True}}, conflicts=True)
self.assertEqual(
result[0]["_conflicts"][0], "1-23202479633c2b380f79507a776743d5"
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 0a52dee..6793601 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -90,7 +90,9 @@ def add_view_indexes(db, kwargs):
(["ordered"], "ordered"),
]
for (idx, name) in indexes:
- assert db.create_index(idx, name=name, ddoc=name) is True
+ assert db.create_index(idx, name=name, ddoc=name,
+ wait_for_built_index=False) is True
+ db.wait_for_built_indexes()
def add_text_indexes(db, kwargs):
[couchdb] 01/02: add bookmark support
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5d753b61a1122ea32e3ff927630549476df24906
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Feb 10 15:21:42 2020 +0200
add bookmark support
---
src/mango/src/mango_cursor_view.erl | 48 ++++++++++++++++++++++++-------------
src/mango/src/mango_fdb.erl | 45 ++++++++++++++++++----------------
src/mango/src/mango_idx.erl | 1 -
src/mango/test/mango.py | 15 ++++++++++--
4 files changed, 70 insertions(+), 39 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 15eb55d..54107ec 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -99,6 +99,9 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
maybe_replace_max_json(EndKey) ->
EndKey.
+%% TODO: When supported, handle:
+%% partitions
+%% conflicts
index_args(#cursor{} = Cursor) ->
#cursor{
index = Idx,
@@ -106,6 +109,7 @@ index_args(#cursor{} = Cursor) ->
bookmark = Bookmark
} = Cursor,
io:format("SELE ~p ranges ~p ~n", [Cursor#cursor.selector, Cursor#cursor.ranges]),
+
Args0 = #{
start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
start_key_docid => <<>>,
@@ -113,19 +117,27 @@ index_args(#cursor{} = Cursor) ->
end_key_docid => <<255>>,
skip => 0
},
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
Sort = couch_util:get_value(sort, Opts, [<<"asc">>]),
- io:format("SORT ~p ~n", [Sort]),
Args1 = case mango_sort:directions(Sort) of
- [<<"desc">> | _] -> Args#{dir => rev};
- _ -> Args#{dir => fwd}
+ [<<"desc">> | _] ->
+ #{
+ start_key := SK,
+ start_key_docid := SKDI,
+ end_key := EK,
+ end_key_docid := EKDI
+ } = Args0,
+ Args0#{
+ dir => rev,
+ start_key => EK,
+ start_key_docid => EKDI,
+ end_key => SK,
+ end_key_docid => SKDI
+ };
+ _ ->
+ Args0#{dir => fwd}
end,
-
- %% TODO: When supported, handle:
- %% partitions
- %% conflicts
- Args1.
+ mango_json_bookmark:update_args(Bookmark, Args1).
execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
@@ -299,14 +311,14 @@ set_mango_msg_timestamp() ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
-handle_message({doc, Doc}, Cursor) ->
+handle_message({doc, Key, Doc}, Cursor) ->
case doc_member(Cursor, Doc) of
{ok, Doc, {execution_stats, ExecutionStats1}} ->
Cursor1 = Cursor#cursor {
execution_stats = ExecutionStats1
},
{Props} = Doc,
- Cursor2 = update_bookmark_keys(Cursor1, Props),
+ Cursor2 = update_bookmark_keys(Cursor1, {Key, Props}),
FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
handle_doc(Cursor2, FinalDoc);
{no_match, _, {execution_stats, ExecutionStats1}} ->
@@ -329,8 +341,10 @@ handle_all_docs_message({row, Props}, Cursor) ->
case is_design_doc(Props) of
true -> {ok, Cursor};
false ->
- {doc, Doc} = lists:keyfind(doc, 1, Props),
- handle_message({doc, Doc}, Cursor)
+ Doc = couch_util:get_value(doc, Props),
+ Key = couch_util:get_value(key, Props),
+
+ handle_message({doc, Key, Doc}, Cursor)
end;
handle_all_docs_message(Message, Cursor) ->
handle_message(Message, Cursor).
@@ -480,9 +494,11 @@ is_design_doc(RowProps) ->
end.
-update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
+update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props}) when Limit > 0 ->
+ io:format("PROPS ~p ~n", [Props]),
+ Id = couch_util:get_value(<<"_id">>, Props),
+%% Key = couch_util:get_value(<<"key">>, Props),
+ io:format("BOOMARK KEYS id ~p key ~p ~n", [Id, Key]),
Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 3e0c48e..9a17a85 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -121,7 +121,8 @@ write_doc(TxDb, DocId, IdxResults) ->
query_all_docs(Db, CallBack, Cursor, Args) ->
- Opts = args_to_fdb_opts(Args) ++ [{include_docs, true}],
+ Opts = args_to_fdb_opts(Args, true) ++ [{include_docs, true}],
+ io:format("ALL DOC OPTS ~p ~n", [Opts]),
fabric2_db:fold_docs(Db, CallBack, Cursor, Opts).
@@ -138,7 +139,7 @@ query(Db, CallBack, Cursor, Args) ->
callback => CallBack
},
- Opts = args_to_fdb_opts(Args),
+ Opts = args_to_fdb_opts(Args, false),
io:format("OPTS ~p ~n", [Opts]),
try
Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
@@ -153,7 +154,7 @@ query(Db, CallBack, Cursor, Args) ->
end).
-args_to_fdb_opts(Args) ->
+args_to_fdb_opts(Args, AllDocs) ->
#{
start_key := StartKey0,
start_key_docid := StartKeyDocId,
@@ -165,16 +166,6 @@ args_to_fdb_opts(Args) ->
io:format("ARGS ~p ~n", [Args]),
io:format("START ~p ~n End ~p ~n", [StartKey0, EndKey0]),
-%% StartKey1 = if StartKey0 == undefined -> undefined; true ->
-%% couch_views_encoding:encode(StartKey0, key)
-%% end,
-
- % fabric2_fdb:fold_range switches keys around because map/reduce switches them
- % but we do need to switch them. So we do this fun dance
- {StartKeyName, EndKeyName} = case Direction of
- rev -> {end_key, start_key};
- _ -> {start_key, end_key}
- end,
StartKeyOpts = case {StartKey0, StartKeyDocId} of
{[], _} ->
@@ -182,11 +173,16 @@ args_to_fdb_opts(Args) ->
{null, _} ->
%% all_docs no startkey
[];
-%% {undefined, _} ->
-%% [];
+ {StartKey0, _} when AllDocs == true ->
+ StartKey1 = if is_binary(StartKey0) -> StartKey0; true ->
+ %% couch_views_encoding:encode(StartKey0, key)
+ couch_util:to_binary(StartKey0)
+ end,
+ io:format("START SEction ~p ~n", [StartKey1]),
+ [{start_key, StartKey1}];
{StartKey0, StartKeyDocId} ->
StartKey1 = couch_views_encoding:encode(StartKey0, key),
- [{StartKeyName, {StartKey1, StartKeyDocId}}]
+ [{start_key, {StartKey1, StartKeyDocId}}]
end,
InclusiveEnd = true,
@@ -201,11 +197,18 @@ args_to_fdb_opts(Args) ->
{[<<255>>], _, _} ->
%% mango index no endkey with a $lt in selector
[];
+ {EndKey0, EndKeyDocId, _} when AllDocs == true ->
+ EndKey1 = if is_binary(EndKey0) -> EndKey0; true ->
+ couch_util:to_binary(EndKey0)
+ end,
+ io:format("ENDKEY ~p ~n", [EndKey1]),
+ [{end_key, EndKey1}];
{EndKey0, EndKeyDocId, _} when InclusiveEnd ->
EndKey1 = couch_views_encoding:encode(EndKey0, key),
- [{EndKeyName, {EndKey1, EndKeyDocId}}]
+ [{end_key, {EndKey1, EndKeyDocId}}]
end,
+
[
{skip, Skip},
{dir, Direction},
@@ -213,7 +216,7 @@ args_to_fdb_opts(Args) ->
] ++ StartKeyOpts ++ EndKeyOpts.
-fold_cb({Key, _}, Acc) ->
+fold_cb({Key, Val}, Acc) ->
#{
prefix := MangoIdxPrefix,
db := Db,
@@ -222,10 +225,11 @@ fold_cb({Key, _}, Acc) ->
} = Acc,
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+ SortKeys = couch_views_encoding:decode(Val),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
JSONDoc = couch_doc:to_json_obj(Doc, []),
io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
- case Callback({doc, JSONDoc}, Cursor) of
+ case Callback({doc, SortKeys, JSONDoc}, Cursor) of
{ok, Cursor1} ->
Acc#{
cursor := Cursor1
@@ -274,5 +278,6 @@ add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
tx := Tx
} = TxDb,
Key = create_key(MangoIdxPrefix, Results, DocId),
- erlfdb:set(Tx, Key, <<0>>).
+ Val = couch_views_encoding:encode(Results),
+ erlfdb:set(Tx, Key, Val).
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index c1deaa9..5be8530 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -322,7 +322,6 @@ start_key(#idx{}=Idx, Ranges) ->
end_key(#idx{}=Idx, Ranges) ->
Mod = idx_mod(Idx),
- io:format("END KEY ~p ~n", [Mod]),
Mod:end_key(Ranges).
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 5b1c7a7..62d6c1b 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -138,7 +138,8 @@ class Database(object):
name=None,
ddoc=None,
partial_filter_selector=None,
- selector=None
+ selector=None,
+ wait_for_built_index=True
):
body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
if name is not None:
@@ -156,7 +157,7 @@ class Database(object):
assert r.json()["name"] is not None
created = r.json()["result"] == "created"
- if created:
+ if created and wait_for_built_index:
# wait until the database reports the index as available and build
while len([
i
@@ -167,6 +168,16 @@ class Database(object):
return created
+ def wait_for_built_indexes(self):
+ indexes = self.list_indexes()
+ while len([
+ i
+ for i in self.list_indexes()
+ if i["build_status"] == "ready"
+ ]) < len(indexes):
+ delay(t=0.2)
+
+
def create_text_index(
self,
analyzer=None,