You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/01/27 12:54:09 UTC
[couchdb] branch fdb-mango-indexes updated: temp commit
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/fdb-mango-indexes by this push:
new 42b79f4 temp commit
42b79f4 is described below
commit 42b79f486c2456a93d5a6c01c9d5556f002b4684
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Jan 27 14:53:37 2020 +0200
temp commit
---
src/mango/src/mango_cursor_view.erl | 225 ++++++++++++++++++----------------
src/mango/src/mango_fdb.erl | 60 ++++++++-
src/mango/src/mango_indexer.erl | 9 +-
src/mango/src/mango_json_bookmark.erl | 24 ++--
src/mango/test/02-basic-find-test.py | 13 +-
src/mango/test/user_docs.py | 6 +-
6 files changed, 205 insertions(+), 132 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 7b47a40..06a8b40 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,7 +19,7 @@
]).
-export([
- view_cb/2,
+%% view_cb/2,
handle_message/2,
handle_all_docs_message/2,
composite_indexes/2,
@@ -63,22 +63,23 @@ explain(Cursor) ->
#cursor{
opts = Opts
} = Cursor,
-
- BaseArgs = base_args(Cursor),
- Args = apply_opts(Opts, BaseArgs),
-
- [{mrargs, {[
- {include_docs, Args#mrargs.include_docs},
- {view_type, Args#mrargs.view_type},
- {reduce, Args#mrargs.reduce},
- {partition, couch_mrview_util:get_extra(Args, partition, null)},
- {start_key, maybe_replace_max_json(Args#mrargs.start_key)},
- {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
- {direction, Args#mrargs.direction},
- {stable, Args#mrargs.stable},
- {update, Args#mrargs.update},
- {conflicts, Args#mrargs.conflicts}
- ]}}].
+ [{marargs, {[]}}].
+
+%% BaseArgs = base_args(Cursor),
+%% Args = apply_opts(Opts, BaseArgs),
+%%
+%% [{mrargs, {[
+%% {include_docs, Args#mrargs.include_docs},
+%% {view_type, Args#mrargs.view_type},
+%% {reduce, Args#mrargs.reduce},
+%% {partition, couch_mrview_util:get_extra(Args, partition, null)},
+%% {start_key, maybe_replace_max_json(Args#mrargs.start_key)},
+%% {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
+%% {direction, Args#mrargs.direction},
+%% {stable, Args#mrargs.stable},
+%% {update, Args#mrargs.update},
+%% {conflicts, Args#mrargs.conflicts}
+%% ]}}].
% replace internal values that cannot
@@ -99,15 +100,28 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
- #mrargs{
- view_type = map,
- reduce = false,
- start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges),
- end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
- include_docs = true,
- extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
- }.
+index_args(#cursor{index = Idx} = Cursor) ->
+ #cursor{
+ index = Idx,
+ opts = Opts,
+ bookmark = Bookmark
+ } = Cursor,
+ Args0 = #{
+ start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
+ end_key => mango_idx:end_key(Idx, Cursor#cursor.ranges)
+ },
+ Args = mango_json_bookmark:update_args(Bookmark, Args0),
+
+ Sort = lists:keymember(sort, 1, Opts),
+ Args1 = case mango_sort:directions(Sort) of
+ [<<"desc">> | _] -> Args#{reverse => true};
+ _ -> Args#{reverse => false}
+ end,
+
+ %% TODO: When supported handle:
+ %% partitions
+ %% conflicts
+ Args1.
execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
@@ -121,21 +135,23 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
% empty indicates unsatisfiable ranges, so don't perform search
{ok, UserAcc};
_ ->
- BaseArgs = base_args(Cursor),
+ Args = index_args(Cursor),
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
- Args0 = apply_opts(Opts, BaseArgs),
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
+%% Args0 = BaseArgs,
+%% Args0 = apply_opts(Opts, BaseArgs),
+%% Args = mango_json_bookmark:update_args(Bookmark, Args0),
UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
all_docs ->
CB = fun ?MODULE:handle_all_docs_message/2,
- fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+ ok;
+%% fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
_ ->
CB = fun ?MODULE:handle_message/2,
% Normal view
- DDoc = ddocid(Idx),
- Name = mango_idx:name(Idx),
+%% DDoc = ddocid(Idx),
+%% Name = mango_idx:name(Idx),
mango_fdb:query(Db, CB, Cursor, Args)
end,
case Result of
@@ -343,72 +359,72 @@ ddocid(Idx) ->
end.
-apply_opts([], Args) ->
- Args;
-apply_opts([{r, RStr} | Rest], Args) ->
- IncludeDocs = case list_to_integer(RStr) of
- 1 ->
- true;
- R when R > 1 ->
- % We don't load the doc in the view query because
- % we have to do a quorum read in the coordinator
- % so there's no point.
- false
- end,
- NewArgs = Args#mrargs{include_docs = IncludeDocs},
- apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, true} | Rest], Args) ->
- NewArgs = Args#mrargs{conflicts = true},
- apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, false} | Rest], Args) ->
- % Ignored cause default
- apply_opts(Rest, Args);
-apply_opts([{sort, Sort} | Rest], Args) ->
- % We only support single direction sorts
- % so nothing fancy here.
- case mango_sort:directions(Sort) of
- [] ->
- apply_opts(Rest, Args);
- [<<"asc">> | _] ->
- apply_opts(Rest, Args);
- [<<"desc">> | _] ->
- SK = Args#mrargs.start_key,
- SKDI = Args#mrargs.start_key_docid,
- EK = Args#mrargs.end_key,
- EKDI = Args#mrargs.end_key_docid,
- NewArgs = Args#mrargs{
- direction = rev,
- start_key = EK,
- start_key_docid = EKDI,
- end_key = SK,
- end_key_docid = SKDI
- },
- apply_opts(Rest, NewArgs)
- end;
-apply_opts([{stale, ok} | Rest], Args) ->
- NewArgs = Args#mrargs{
- stable = true,
- update = false
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{stable, true} | Rest], Args) ->
- NewArgs = Args#mrargs{
- stable = true
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{update, false} | Rest], Args) ->
- NewArgs = Args#mrargs{
- update = false
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{partition, <<>>} | Rest], Args) ->
- apply_opts(Rest, Args);
-apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
- NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
- apply_opts(Rest, NewArgs);
-apply_opts([{_, _} | Rest], Args) ->
- % Ignore unknown options
- apply_opts(Rest, Args).
+%%apply_opts([], Args) ->
+%% Args;
+%%apply_opts([{r, RStr} | Rest], Args) ->
+%% IncludeDocs = case list_to_integer(RStr) of
+%% 1 ->
+%% true;
+%% R when R > 1 ->
+%% % We don't load the doc in the view query because
+%% % we have to do a quorum read in the coordinator
+%% % so there's no point.
+%% false
+%% end,
+%% NewArgs = Args#mrargs{include_docs = IncludeDocs},
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{conflicts, true} | Rest], Args) ->
+%% NewArgs = Args#mrargs{conflicts = true},
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{conflicts, false} | Rest], Args) ->
+%% % Ignored cause default
+%% apply_opts(Rest, Args);
+%%apply_opts([{sort, Sort} | Rest], Args) ->
+%% % We only support single direction sorts
+%% % so nothing fancy here.
+%% case mango_sort:directions(Sort) of
+%% [] ->
+%% apply_opts(Rest, Args);
+%% [<<"asc">> | _] ->
+%% apply_opts(Rest, Args);
+%% [<<"desc">> | _] ->
+%% SK = Args#mrargs.start_key,
+%% SKDI = Args#mrargs.start_key_docid,
+%% EK = Args#mrargs.end_key,
+%% EKDI = Args#mrargs.end_key_docid,
+%% NewArgs = Args#mrargs{
+%% direction = rev,
+%% start_key = EK,
+%% start_key_docid = EKDI,
+%% end_key = SK,
+%% end_key_docid = SKDI
+%% },
+%% apply_opts(Rest, NewArgs)
+%% end;
+%%apply_opts([{stale, ok} | Rest], Args) ->
+%% NewArgs = Args#mrargs{
+%% stable = true,
+%% update = false
+%% },
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{stable, true} | Rest], Args) ->
+%% NewArgs = Args#mrargs{
+%% stable = true
+%% },
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{update, false} | Rest], Args) ->
+%% NewArgs = Args#mrargs{
+%% update = false
+%% },
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{partition, <<>>} | Rest], Args) ->
+%% apply_opts(Rest, Args);
+%%apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
+%% NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
+%% apply_opts(Rest, NewArgs);
+%%apply_opts([{_, _} | Rest], Args) ->
+%% % Ignore unknown options
+%% apply_opts(Rest, Args).
doc_member(Cursor, DocProps) ->
@@ -448,13 +464,12 @@ doc_member(Cursor, DocProps) ->
match_doc(Selector, Doc, ExecutionStats) ->
- {ok, Doc, {execution_stats, ExecutionStats}}.
-%% case mango_selector:match(Selector, Doc) of
-%% true ->
-%% {ok, Doc, {execution_stats, ExecutionStats}};
-%% false ->
-%% {no_match, Doc, {execution_stats, ExecutionStats}}
-%% end.
+ case mango_selector:match(Selector, Doc) of
+ true ->
+ {ok, Doc, {execution_stats, ExecutionStats}};
+ false ->
+ {no_match, Doc, {execution_stats, ExecutionStats}}
+ end.
is_design_doc(RowProps) ->
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index 36cb400..4675f54 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -38,8 +38,10 @@ query(Db, CallBack, Cursor, Args) ->
db => TxDb,
callback => CallBack
},
+
+ Opts = args_to_fdb_opts(Args),
io:format("DB ~p ~n", [TxDb]),
- Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, []),
+ Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
#{
cursor := Cursor1
} = Acc1,
@@ -47,6 +49,62 @@ query(Db, CallBack, Cursor, Args) ->
end).
+args_to_fdb_opts(Args) ->
+ #{
+ start_key := StartKey0,
+ start_key_docid := StartKeyDocId,
+ end_key := EndKey0,
+ end_key_docid := EndKeyDocId,
+ reverse = Reverse,
+ limit = Limit,
+ skip = Skip
+ } = Args,
+
+ StartKey1 = if StartKey0 == undefined -> undefined; true ->
+ couch_views_encoding:encode(StartKey0, key)
+ end,
+
+ StartKeyOpts = case {StartKey1, StartKeyDocId} of
+ {undefined, _} ->
+ [];
+ {StartKey1, StartKeyDocId} ->
+ [{start_key, {StartKey1, StartKeyDocId}}]
+ end,
+
+ EndKey1 = if EndKey0 == undefined -> undefined; true ->
+ couch_views_encoding:encode(EndKey0, key)
+ end,
+
+ EndKeyOpts = case {EndKey1, EndKeyDocId, Reverse} of
+ {undefined, _, _} ->
+ [];
+ {EndKey1, <<>>, rev} when not InclusiveEnd ->
+ % When we iterate in reverse with
+ % inclusive_end=false we have to set the
+ % EndKeyDocId to <<255>> so that we don't
+ % include matching rows.
+ [{end_key_gt, {EndKey1, <<255>>}}];
+ {EndKey1, <<255>>, _} when not InclusiveEnd ->
+ % When inclusive_end=false we need to
+ % elide the default end_key_docid so as
+ % to not sort past the docids with the
+ % given end key.
+ [{end_key_gt, {EndKey1}}];
+ {EndKey1, EndKeyDocId, _} when not InclusiveEnd ->
+ [{end_key_gt, {EndKey1, EndKeyDocId}}];
+ {EndKey1, EndKeyDocId, _} when InclusiveEnd ->
+ [{end_key, {EndKey1, EndKeyDocId}}]
+ end,
+
+ [
+ {dir, Direction},
+ {limit, Limit * 2 + Skip * 2},
+ {streaming_mode, want_all}
+ ] ++ StartKeyOpts ++ EndKeyOpts.
+
+ ok.
+
+
fold_cb({Key, _}, Acc) ->
#{
prefix := MangoIdxPrefix,
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index 2040059..36eb2d3 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -37,7 +37,7 @@ update(Db, updated, Doc, OldDoc) ->
update(Db, created, Doc, _) ->
#doc{id = DocId} = Doc,
Indexes = mango_idx:list(Db),
- Indexes1 = filter_and_to_json(Indexes),
+ Indexes1 = filter_json_indexes(Indexes),
io:format("UPDATE INDEXES ~p ~n filtered ~p ~n", [Indexes, Indexes1]),
JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
io:format("DOC ~p ~n", [Doc]),
@@ -46,12 +46,9 @@ update(Db, created, Doc, _) ->
mango_fdb:write_doc(Db, DocId, Results).
-filter_and_to_json(Indexes) ->
+filter_json_indexes(Indexes) ->
lists:filter(fun (Idx) ->
- case Idx#idx.type == <<"special">> of
- true -> false;
- false -> true
- end
+ Idx#idx.type == <<"json">>
end, Indexes).
diff --git a/src/mango/src/mango_json_bookmark.erl b/src/mango/src/mango_json_bookmark.erl
index 97f81cf..edc83cf 100644
--- a/src/mango/src/mango_json_bookmark.erl
+++ b/src/mango/src/mango_json_bookmark.erl
@@ -23,23 +23,28 @@
-include("mango_cursor.hrl").
-include("mango.hrl").
-update_args(EncodedBookmark, #mrargs{skip = Skip} = Args) ->
+update_args(EncodedBookmark, FdbOpts) ->
+ #{
+ skip := Skip
+ } = FdbOpts,
Bookmark = unpack(EncodedBookmark),
case is_list(Bookmark) of
true ->
{startkey, Startkey} = lists:keyfind(startkey, 1, Bookmark),
- {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1, Bookmark),
- Args#mrargs{
- start_key = Startkey,
- start_key_docid = StartkeyDocId,
- skip = 1 + Skip
+ {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1,
+ Bookmark),
+ FdbOpts#{
+ start_key => Startkey,
+ start_key_docid => StartkeyDocId,
+ skip => 1 + Skip
};
false ->
- Args
+ FdbOpts
end.
-create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey}) when BookmarkKey =/= undefined ->
+create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey})
+ when BookmarkKey =/= undefined ->
QueryArgs = [
{startkey_docid, BookmarkDocId},
{startkey, BookmarkKey}
@@ -61,7 +66,8 @@ unpack(Packed) ->
end.
verify(Bookmark) when is_list(Bookmark) ->
- case lists:keymember(startkey, 1, Bookmark) andalso lists:keymember(startkey_docid, 1, Bookmark) of
+ case lists:keymember(startkey, 1, Bookmark)
+ andalso lists:keymember(startkey_docid, 1, Bookmark) of
true -> Bookmark;
_ -> throw(invalid_bookmark)
end;
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 632ad4f..8c2a9b1 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -118,16 +118,13 @@ class BasicFindTests(mango.UserDocsTests):
# assert e.response.status_code == 400
# else:
# raise AssertionError("bad find")
- #
+
def test_simple_find(self):
- print("OK")
- docs = self.db.find({"age": {"$lt": 45}})
- print("DOC")
- print(docs)
+ docs = self.db.find({"age": {"$lt": 35}})
assert len(docs) == 3
- # assert docs[0]["user_id"] == 9
- # assert docs[1]["user_id"] == 1
- # assert docs[2]["user_id"] == 7
+ assert docs[0]["user_id"] == 9
+ assert docs[1]["user_id"] == 1
+ assert docs[2]["user_id"] == 7
# def test_multi_cond_and(self):
# docs = self.db.find({"manager": True, "location.city": "Longbranch"})
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 45fbd24..c021f66 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -65,9 +65,9 @@ def setup(db, index_type="view", **kwargs):
add_view_indexes(db, kwargs)
elif index_type == "text":
add_text_indexes(db, kwargs)
- copy_docs = copy.deepcopy(DOCS)
- resp = db.save_doc(copy_docs[0])
- # db.save_docs(copy.deepcopy(DOCS))
+ # copy_docs = copy.deepcopy(DOCS)
+ # resp = db.save_doc(copy_docs[0])
+ db.save_docs(copy.deepcopy(DOCS))
def add_view_indexes(db, kwargs):