You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/04/02 12:39:43 UTC
[couchdb] 15/18: Update Mango query to work with couch_views
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5fc19004a42b9ba2ae55cc17fa1b7840374dff47
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:21:22 2020 +0200
Update Mango query to work with couch_views
---
src/couch_views/test/couch_views_map_test.erl | 41 +++++++++-
src/mango/src/mango_crud.erl | 24 ++----
src/mango/src/mango_cursor.erl | 4 +-
src/mango/src/mango_cursor_view.erl | 70 +++++------------
src/mango/src/mango_httpd.erl | 105 ++++++++++++++------------
src/mango/src/mango_idx.erl | 13 +++-
src/mango/src/mango_idx_view.erl | 5 +-
src/mango/src/mango_idx_view.hrl | 13 ----
src/mango/src/mango_util.erl | 11 +--
9 files changed, 139 insertions(+), 147 deletions(-)
diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl
index f8ba183..7d1e94b 100644
--- a/src/couch_views/test/couch_views_map_test.erl
+++ b/src/couch_views/test/couch_views_map_test.erl
@@ -14,6 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
-define(TDEF(A), {atom_to_list(A), fun A/0}).
@@ -56,7 +57,8 @@ map_views_test_() ->
?TDEF(should_map_duplicate_keys),
?TDEF(should_map_with_doc_emit),
?TDEF(should_map_update_is_false),
- ?TDEF(should_map_update_is_lazy)
+ ?TDEF(should_map_update_is_lazy),
+ ?TDEF(should_map_wait_for_interactive)
% fun should_give_ext_size_seq_indexed_test/1
]
}
@@ -419,6 +421,25 @@ should_map_update_is_lazy() ->
?assertEqual(Expect, Result2).
+should_map_wait_for_interactive() ->
+ DbName = ?tempdb(),
+ {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_interactive_ddoc(),
+ Docs = make_docs(101),
+
+ fabric2_db:update_docs(Db, Docs),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Result = couch_views:query(Db, DDoc, <<"idx_01">>, fun default_cb/2, [],
+ #{limit => 3}),
+ ?assertEqual({ok, [
+ {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}
+ ]}, Result).
+
+
% should_give_ext_size_seq_indexed_test(Db) ->
% DDoc = couch_doc:from_json_obj({[
% {<<"_id">>, <<"_design/seqdoc">>},
@@ -510,6 +531,24 @@ create_ddoc() ->
]}}
]}).
+create_interactive_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc_interactive">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, <<
+ "function(doc) {"
+ "if (doc.val) {"
+ "emit(doc.val, doc.val);"
+ "}"
+ "}">>}
+ ]}}
+ ]}},
+ {<<"autoupdate">>, false},
+ {<<"interactive">>, true}
+ ]}).
+
make_docs(Count) ->
[doc(I) || I <- lists:seq(1, Count)].
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 42717ff..66cef65 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -33,9 +33,8 @@ insert(Db, #doc{}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
-insert(Db, Docs, Opts0) when is_list(Docs) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- case fabric:update_docs(Db, Docs, Opts1) of
+insert(Db, Docs, Opts) when is_list(Docs) ->
+ case fabric2_db:update_docs(Db, Docs, Opts) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -45,9 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
end.
-find(Db, Selector, Callback, UserAcc, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+find(Db, Selector, Callback, UserAcc, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -97,21 +95,11 @@ delete(Db, Selector, Options) ->
end.
-explain(Db, Selector, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+explain(Db, Selector, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:explain(Cursor).
-maybe_add_user_ctx(Db, Opts) ->
- case lists:keyfind(user_ctx, 1, Opts) of
- {user_ctx, _} ->
- Opts;
- false ->
- [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
- end.
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index db4e981..63b449c 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -48,7 +48,9 @@
create(Db, Selector0, Opts) ->
Selector = mango_selector:normalize(Selector0),
- UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+ UsableIndexes = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ mango_idx:get_usable_indexes(TxDb, Selector, Opts)
+ end),
case mango_cursor:maybe_filter_indexes_by_ddoc(UsableIndexes, Opts) of
[] ->
% use_index doesn't match a valid index - fall back to a valid one
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index b88f6ea..4960fa1 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -31,9 +31,7 @@
-include_lib("fabric/include/fabric.hrl").
-include("mango_cursor.hrl").
--include("mango_idx_view.hrl").
--define(HEARTBEAT_INTERVAL_IN_USEC, 4000000).
create(Db, Indexes, Selector, Opts) ->
FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -91,7 +89,8 @@ maybe_replace_max_json(?MAX_STR) ->
<<"<MAX>">>;
maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
- H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+ MAX_VAL = couch_views_encoding:max(),
+ H1 = if H == MAX_VAL -> <<"<MAX>">>;
true -> H
end,
[H1 | maybe_replace_max_json(T)];
@@ -100,7 +99,7 @@ maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
+base_args(#cursor{index = Idx} = Cursor) ->
{StartKey, EndKey} = case Cursor#cursor.ranges of
[empty] ->
{null, null};
@@ -132,18 +131,19 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
Args = mango_json_bookmark:update_args(Bookmark, Args0),
- UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
- DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
all_docs ->
CB = fun ?MODULE:handle_all_docs_message/2,
- fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+ AllDocOpts = fabric2_util:all_docs_view_opts(Args)
+ ++ [{restart_tx, true}],
+ fabric2_db:fold_docs(Db, CB, Cursor, AllDocOpts);
_ ->
CB = fun ?MODULE:handle_message/2,
% Normal view
- DDoc = ddocid(Idx),
+ DDocId = mango_idx:ddoc(Idx),
+ {ok, DDoc} = fabric2_db:open_doc(Db, DDocId),
Name = mango_idx:name(Idx),
- fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args)
+ couch_views:query(Db, DDoc, Name, CB, Cursor, Args)
end,
case Result of
{ok, LastCursor} ->
@@ -227,7 +227,7 @@ choose_best_index(_DbName, IndexRanges) ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
- case doc_member(Cursor, Props) of
+ case match_doc(Cursor, Props) of
{ok, Doc, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
execution_stats = Stats
@@ -280,15 +280,6 @@ handle_doc(C, _Doc) ->
{stop, C}.
-ddocid(Idx) ->
- case mango_idx:ddoc(Idx) of
- <<"_design/", Rest/binary>> ->
- Rest;
- Else ->
- Else
- end.
-
-
apply_opts([], Args) ->
Args;
apply_opts([{conflicts, true} | Rest], Args) ->
@@ -340,41 +331,18 @@ apply_opts([{_, _} | Rest], Args) ->
apply_opts(Rest, Args).
-doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
- Opts = Cursor#cursor.opts,
- ExecutionStats = Cursor#cursor.execution_stats,
- Selector = Cursor#cursor.selector,
- case couch_util:get_value(doc, RowProps) of
- {DocProps} ->
- % only matching documents are returned; the selector
- % is evaluated at the shard level in view_cb({row, Row},
- {ok, {DocProps}, {execution_stats, ExecutionStats}};
- undefined ->
- % an undefined doc was returned, indicating we should
- % perform a quorum fetch
- ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
- couch_stats:increment_counter([mango, quorum_docs_examined]),
- Id = couch_util:get_value(id, RowProps),
- case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
- {ok, #doc{}=DocProps} ->
- Doc = couch_doc:to_json_obj(DocProps, []),
- match_doc(Selector, Doc, ExecutionStats1);
- Else ->
- Else
- end;
- _ ->
- % no doc, no match
- {no_match, null, {execution_stats, ExecutionStats}}
- end.
-
-
-match_doc(Selector, Doc, ExecutionStats) ->
+match_doc(Cursor, RowProps) ->
+ #cursor{
+ execution_stats = Stats0,
+ selector = Selector
+ } = Cursor,
+ Stats1 = mango_execution_stats:incr_docs_examined(Stats0, 1),
+ Doc = couch_util:get_value(doc, RowProps),
case mango_selector:match(Selector, Doc) of
true ->
- {ok, Doc, {execution_stats, ExecutionStats}};
+ {ok, Doc, {execution_stats, Stats1}};
false ->
- {no_match, Doc, {execution_stats, ExecutionStats}}
+ {no_match, Doc, {execution_stats, Stats1}}
end.
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 1054c74..94aa866 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,9 +32,8 @@
threshold = 1490
}).
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
try
- Db = set_user_ctx(Req, Db0),
handle_req_int(Req, Db)
catch
throw:{mango_error, Module, Reason} ->
@@ -61,7 +60,9 @@ handle_req_int(_, _) ->
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
- Idxs = lists:sort(mango_idx:list(Db)),
+ Idxs = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ lists:sort(mango_idx:list(TxDb))
+ end),
JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
TotalRows = length(JsonIdxs0),
Limit = case couch_util:get_value(limit, Params, TotalRows) of
@@ -87,25 +88,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, Idx0} = mango_idx:new(Db, Opts),
{ok, Idx} = mango_idx:validate_new(Idx0, Db),
DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body],
- {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts),
- Id = Idx#idx.ddoc,
- Name = Idx#idx.name,
- Status = case mango_idx:add(DDoc, Idx) of
- {ok, DDoc} ->
- <<"exists">>;
- {ok, NewDDoc} ->
- case mango_crud:insert(Db, NewDDoc, Opts) of
- {ok, [{RespProps}]} ->
- case lists:keyfind(error, 1, RespProps) of
- {error, Reason} ->
- ?MANGO_ERROR({error_saving_ddoc, Reason});
- _ ->
- <<"created">>
- end;
- _ ->
- ?MANGO_ERROR(error_saving_ddoc)
- end
- end,
+ Id = mango_idx:ddoc(Idx),
+ Name = mango_idx:name(Idx),
+ Status = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ {ok, DDoc} = mango_util:load_ddoc(TxDb, Id, DbOpts),
+ case mango_idx:add(DDoc, Idx) of
+ {ok, DDoc} ->
+ <<"exists">>;
+ {ok, NewDDoc} ->
+ case mango_crud:insert(TxDb, NewDDoc, Opts) of
+ {ok, [{RespProps}]} ->
+ case lists:keyfind(error, 1, RespProps) of
+ {error, Reason} ->
+ ?MANGO_ERROR({error_saving_ddoc, Reason});
+ _ ->
+ <<"created">>
+ end;
+ _ ->
+ ?MANGO_ERROR(error_saving_ddoc)
+ end
+ end
+ end),
chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) ->
@@ -118,19 +121,21 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
<<"_bulk_delete">>]}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
- Idxs = mango_idx:list(Db),
- DDocs = get_bulk_delete_ddocs(Opts),
- {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
- Id = {<<"id">>, DDocId},
- case mango_idx:delete(Filt, Db, Idxs, Opts) of
- {ok, true} ->
- {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
- {error, Error} ->
- {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
- end
- end, {[], []}, DDocs),
+ {Success, Fail} = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocs = get_bulk_delete_ddocs(Opts),
+ lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
+ Id = {<<"id">>, DDocId},
+ case mango_idx:delete(Filt, TxDb, Idxs, Opts) of
+ {ok, true} ->
+ {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
+ {error, Error} ->
+ {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
+ end
+ end, {[], []}, DDocs)
+ end),
chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]});
handle_index_req(#httpd{path_parts=[_, <<"_index">>,
@@ -144,15 +149,18 @@ handle_index_req(#httpd{method='DELETE',
handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
- Idxs = mango_idx:list(Db),
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) ->
- IsDDoc = mango_idx:ddoc(Idx) == DDocId,
- IsType = mango_idx:type(Idx) == Type,
- IsName = mango_idx:name(Idx) == Name,
- IsDDoc andalso IsType andalso IsName
- end,
- case mango_idx:delete(Filt, Db, Idxs, []) of
+ Result = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) ->
+ IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+ IsType = mango_idx:type(Idx) == Type,
+ IsName = mango_idx:name(Idx) == Name,
+ IsDDoc andalso IsType andalso IsName
+ end,
+ mango_idx:delete(Filt, TxDb, Idxs, [])
+ end),
+ case Result of
{ok, true} ->
chttpd:send_json(Req, {[{ok, true}]});
{error, not_found} ->
@@ -170,7 +178,9 @@ handle_explain_req(#httpd{method='POST'}=Req, Db) ->
Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
- Resp = mango_crud:explain(Db, Sel, Opts),
+ Resp = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ mango_crud:explain(TxDb, Sel, Opts)
+ end),
chttpd:send_json(Req, Resp);
handle_explain_req(Req, _Db) ->
@@ -195,11 +205,6 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
- {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
- NewDb.
-
-
get_bulk_delete_ddocs(Opts) ->
case lists:keyfind(docids, 1, Opts) of
{docids, DDocs} when is_list(DDocs) ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index ba9f68f..99a3588 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -86,7 +86,7 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
- BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
+ BuiltIndexes = remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
@@ -399,6 +399,17 @@ get_legacy_selector(Def) ->
Selector -> Selector
end.
+% remove any interactive indexes that are not built. If an index is not
+% interactive than we do not remove it as it will be built when queried
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun(Idx) ->
+ case Idx#idx.interactive of
+ true -> Idx#idx.build_status == ?INDEX_READY;
+ _ -> true
+ end
+ end, Indexes).
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 84be418..f80cc21 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,7 +34,6 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
--include("mango_idx_view.hrl").
validate_new(#idx{}=Idx, _Db) ->
@@ -183,11 +182,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
false ->
[Key | end_key(Rest)]
end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
deleted file mode 100644
index 0d213e5..0000000
--- a/src/mango/src/mango_idx_view.hrl
+++ /dev/null
@@ -1,13 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 0d31f15..18a6439 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,7 @@ open_doc(Db, DocId) ->
open_doc(Db, DocId, Options) ->
- case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
- {ok, Doc} ->
- {ok, Doc};
- {not_found, _} ->
- not_found;
- _ ->
- ?MANGO_ERROR({error_loading_doc, DocId})
- end.
+ fabric2_db:open_doc(Db, DocId, Options).
open_ddocs(Db) ->
@@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
case open_doc(Db, DDocId, DbOpts) of
{ok, Doc} ->
{ok, check_lang(Doc)};
- not_found ->
+ {not_found, missing} ->
Body = {[
{<<"language">>, <<"query">>}
]},