You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/17 14:32:54 UTC
[couchdb] 14/22: add bookmark support
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fca5b208eb16077387854517a30510cb3d35e234
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Feb 10 15:21:42 2020 +0200
add bookmark support
---
src/mango/src/mango_cursor_view.erl | 48 ++++++++++++++++++++++++-------------
src/mango/src/mango_fdb.erl | 45 ++++++++++++++++++----------------
src/mango/src/mango_idx.erl | 1 -
src/mango/test/mango.py | 15 ++++++++++--
4 files changed, 70 insertions(+), 39 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 15eb55d..54107ec 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -99,6 +99,9 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
maybe_replace_max_json(EndKey) ->
EndKey.
+%% TODO: When supported, handle:
+%% partitions
+%% conflicts
index_args(#cursor{} = Cursor) ->
#cursor{
index = Idx,
@@ -106,6 +109,7 @@ index_args(#cursor{} = Cursor) ->
bookmark = Bookmark
} = Cursor,
io:format("SELE ~p ranges ~p ~n", [Cursor#cursor.selector, Cursor#cursor.ranges]),
+
Args0 = #{
start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
start_key_docid => <<>>,
@@ -113,19 +117,27 @@ index_args(#cursor{} = Cursor) ->
end_key_docid => <<255>>,
skip => 0
},
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
Sort = couch_util:get_value(sort, Opts, [<<"asc">>]),
- io:format("SORT ~p ~n", [Sort]),
Args1 = case mango_sort:directions(Sort) of
- [<<"desc">> | _] -> Args#{dir => rev};
- _ -> Args#{dir => fwd}
+ [<<"desc">> | _] ->
+ #{
+ start_key := SK,
+ start_key_docid := SKDI,
+ end_key := EK,
+ end_key_docid := EKDI
+ } = Args0,
+ Args0#{
+ dir => rev,
+ start_key => EK,
+ start_key_docid => EKDI,
+ end_key => SK,
+ end_key_docid => SKDI
+ };
+ _ ->
+ Args0#{dir => fwd}
end,
-
- %% TODO: When supported, handle:
- %% partitions
- %% conflicts
- Args1.
+ mango_json_bookmark:update_args(Bookmark, Args1).
execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
@@ -299,14 +311,14 @@ set_mango_msg_timestamp() ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
-handle_message({doc, Doc}, Cursor) ->
+handle_message({doc, Key, Doc}, Cursor) ->
case doc_member(Cursor, Doc) of
{ok, Doc, {execution_stats, ExecutionStats1}} ->
Cursor1 = Cursor#cursor {
execution_stats = ExecutionStats1
},
{Props} = Doc,
- Cursor2 = update_bookmark_keys(Cursor1, Props),
+ Cursor2 = update_bookmark_keys(Cursor1, {Key, Props}),
FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
handle_doc(Cursor2, FinalDoc);
{no_match, _, {execution_stats, ExecutionStats1}} ->
@@ -329,8 +341,10 @@ handle_all_docs_message({row, Props}, Cursor) ->
case is_design_doc(Props) of
true -> {ok, Cursor};
false ->
- {doc, Doc} = lists:keyfind(doc, 1, Props),
- handle_message({doc, Doc}, Cursor)
+ Doc = couch_util:get_value(doc, Props),
+ Key = couch_util:get_value(key, Props),
+
+ handle_message({doc, Key, Doc}, Cursor)
end;
handle_all_docs_message(Message, Cursor) ->
handle_message(Message, Cursor).
@@ -480,9 +494,11 @@ is_design_doc(RowProps) ->
end.
-update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
+update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props}) when Limit > 0 ->
+ io:format("PROPS ~p ~n", [Props]),
+ Id = couch_util:get_value(<<"_id">>, Props),
+%% Key = couch_util:get_value(<<"key">>, Props),
+ io:format("BOOMARK KEYS id ~p key ~p ~n", [Id, Key]),
Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 3e0c48e..9a17a85 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -121,7 +121,8 @@ write_doc(TxDb, DocId, IdxResults) ->
query_all_docs(Db, CallBack, Cursor, Args) ->
- Opts = args_to_fdb_opts(Args) ++ [{include_docs, true}],
+ Opts = args_to_fdb_opts(Args, true) ++ [{include_docs, true}],
+ io:format("ALL DOC OPTS ~p ~n", [Opts]),
fabric2_db:fold_docs(Db, CallBack, Cursor, Opts).
@@ -138,7 +139,7 @@ query(Db, CallBack, Cursor, Args) ->
callback => CallBack
},
- Opts = args_to_fdb_opts(Args),
+ Opts = args_to_fdb_opts(Args, false),
io:format("OPTS ~p ~n", [Opts]),
try
Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
@@ -153,7 +154,7 @@ query(Db, CallBack, Cursor, Args) ->
end).
-args_to_fdb_opts(Args) ->
+args_to_fdb_opts(Args, AllDocs) ->
#{
start_key := StartKey0,
start_key_docid := StartKeyDocId,
@@ -165,16 +166,6 @@ args_to_fdb_opts(Args) ->
io:format("ARGS ~p ~n", [Args]),
io:format("START ~p ~n End ~p ~n", [StartKey0, EndKey0]),
-%% StartKey1 = if StartKey0 == undefined -> undefined; true ->
-%% couch_views_encoding:encode(StartKey0, key)
-%% end,
-
- % fabric2_fdb:fold_range switches keys around because map/reduce switches them
- % but we do need to switch them. So we do this fun dance
- {StartKeyName, EndKeyName} = case Direction of
- rev -> {end_key, start_key};
- _ -> {start_key, end_key}
- end,
StartKeyOpts = case {StartKey0, StartKeyDocId} of
{[], _} ->
@@ -182,11 +173,16 @@ args_to_fdb_opts(Args) ->
{null, _} ->
%% all_docs no startkey
[];
-%% {undefined, _} ->
-%% [];
+ {StartKey0, _} when AllDocs == true ->
+ StartKey1 = if is_binary(StartKey0) -> StartKey0; true ->
+ %% couch_views_encoding:encode(StartKey0, key)
+ couch_util:to_binary(StartKey0)
+ end,
+ io:format("START SEction ~p ~n", [StartKey1]),
+ [{start_key, StartKey1}];
{StartKey0, StartKeyDocId} ->
StartKey1 = couch_views_encoding:encode(StartKey0, key),
- [{StartKeyName, {StartKey1, StartKeyDocId}}]
+ [{start_key, {StartKey1, StartKeyDocId}}]
end,
InclusiveEnd = true,
@@ -201,11 +197,18 @@ args_to_fdb_opts(Args) ->
{[<<255>>], _, _} ->
%% mango index no endkey with a $lt in selector
[];
+ {EndKey0, EndKeyDocId, _} when AllDocs == true ->
+ EndKey1 = if is_binary(EndKey0) -> EndKey0; true ->
+ couch_util:to_binary(EndKey0)
+ end,
+ io:format("ENDKEY ~p ~n", [EndKey1]),
+ [{end_key, EndKey1}];
{EndKey0, EndKeyDocId, _} when InclusiveEnd ->
EndKey1 = couch_views_encoding:encode(EndKey0, key),
- [{EndKeyName, {EndKey1, EndKeyDocId}}]
+ [{end_key, {EndKey1, EndKeyDocId}}]
end,
+
[
{skip, Skip},
{dir, Direction},
@@ -213,7 +216,7 @@ args_to_fdb_opts(Args) ->
] ++ StartKeyOpts ++ EndKeyOpts.
-fold_cb({Key, _}, Acc) ->
+fold_cb({Key, Val}, Acc) ->
#{
prefix := MangoIdxPrefix,
db := Db,
@@ -222,10 +225,11 @@ fold_cb({Key, _}, Acc) ->
} = Acc,
{{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+ SortKeys = couch_views_encoding:decode(Val),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
JSONDoc = couch_doc:to_json_obj(Doc, []),
io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
- case Callback({doc, JSONDoc}, Cursor) of
+ case Callback({doc, SortKeys, JSONDoc}, Cursor) of
{ok, Cursor1} ->
Acc#{
cursor := Cursor1
@@ -274,5 +278,6 @@ add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
tx := Tx
} = TxDb,
Key = create_key(MangoIdxPrefix, Results, DocId),
- erlfdb:set(Tx, Key, <<0>>).
+ Val = couch_views_encoding:encode(Results),
+ erlfdb:set(Tx, Key, Val).
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index c1deaa9..5be8530 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -322,7 +322,6 @@ start_key(#idx{}=Idx, Ranges) ->
end_key(#idx{}=Idx, Ranges) ->
Mod = idx_mod(Idx),
- io:format("END KEY ~p ~n", [Mod]),
Mod:end_key(Ranges).
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 5b1c7a7..62d6c1b 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -138,7 +138,8 @@ class Database(object):
name=None,
ddoc=None,
partial_filter_selector=None,
- selector=None
+ selector=None,
+ wait_for_built_index=True
):
body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
if name is not None:
@@ -156,7 +157,7 @@ class Database(object):
assert r.json()["name"] is not None
created = r.json()["result"] == "created"
- if created:
+ if created and wait_for_built_index:
# wait until the database reports the index as available and build
while len([
i
@@ -167,6 +168,16 @@ class Database(object):
return created
+ def wait_for_built_indexes(self):
+ indexes = self.list_indexes()
+ while len([
+ i
+ for i in self.list_indexes()
+ if i["build_status"] == "ready"
+ ]) < len(indexes):
+ delay(t=0.2)
+
+
def create_text_index(
self,
analyzer=None,