You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/04/02 12:39:42 UTC
[couchdb] 14/18: Add mango indexing
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e438a36abeef4d3a6a288a236dfaeb1506e9b2cb
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 15:19:44 2020 +0200
Add mango indexing
This uses couch_views_updater to create mango indexes in the doc update
along with the couch_views_indexer to update the indexes in the
background up to the creation versionstamp.
---
rel/overlay/etc/default.ini | 1 +
src/couch_eval/src/couch_eval.erl | 3 +
src/mango/src/mango_eval.erl | 115 ++++++++++++++++++++++++++++++++++++
src/mango/src/mango_idx.erl | 57 ++++++++++++++----
src/mango/src/mango_idx.hrl | 4 +-
src/mango/src/mango_idx_special.erl | 4 +-
src/mango/src/mango_idx_view.erl | 22 +++++--
7 files changed, 187 insertions(+), 19 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index fd0aa77..d2a2c72 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -355,6 +355,7 @@ os_process_limit = 100
; beahvior for executing provided code in design
; documents.
javascript = couch_js
+query = mango_eval
[mango]
; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl
index 23ca263..3541a5b 100644
--- a/src/couch_eval/src/couch_eval.erl
+++ b/src/couch_eval/src/couch_eval.erl
@@ -75,6 +75,9 @@ acquire_map_context(DbName, DDocId, Language, Sig, Lib, MapFuns) ->
-spec release_map_context(context()) -> ok | {error, any()}.
+release_map_context(nil) ->
+ ok;
+
release_map_context({ApiMod, Ctx}) ->
ApiMod:release_map_context(Ctx).
diff --git a/src/mango/src/mango_eval.erl b/src/mango/src/mango_eval.erl
new file mode 100644
index 0000000..59d784b
--- /dev/null
+++ b/src/mango/src/mango_eval.erl
@@ -0,0 +1,115 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_eval).
+-behavior(couch_eval).
+
+
+-export([
+ acquire_map_context/1,
+ release_map_context/1,
+ map_docs/2
+]).
+
+
+-export([
+ index_doc/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_idx.hrl").
+
+
+acquire_map_context(Opts) ->
+ #{
+ db_name := DbName,
+ ddoc_id := DDocId,
+ map_funs := MapFuns
+ } = Opts,
+ Indexes = lists:map(fun (Def) ->
+ #idx{
+ type = <<"json">>,
+ dbname = DbName,
+ ddoc = DDocId,
+ def = Def
+ }
+ end, MapFuns),
+ {ok, Indexes}.
+
+
+release_map_context(_) ->
+ ok.
+
+
+map_docs(Indexes, Docs) ->
+ {ok, lists:map(fun(Doc) ->
+ Json = couch_doc:to_json_obj(Doc, []),
+ Results = index_doc(Indexes, Json),
+ {Doc#doc.id, Results}
+ end, Docs)}.
+
+
+index_doc(Indexes, Doc) ->
+ lists:map(fun(Idx) ->
+ {IdxDef} = mango_idx:def(Idx),
+ Results = get_index_entries(IdxDef, Doc),
+ case lists:member(not_found, Results) of
+ true ->
+ [];
+ false ->
+ [{Results, null}]
+ end
+ end, Indexes).
+
+
+get_index_entries(IdxDef, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxDef),
+ Selector = get_index_partial_filter_selector(IdxDef),
+ case should_index(Selector, Doc) of
+ false ->
+ [not_found];
+ true ->
+ get_index_values(Fields, Doc)
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxDef) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxDef, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the
+ % partial_filter_selector set as selector
+ couch_util:get_value(<<"selector">>, IdxDef, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index a26a685..ba9f68f 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -50,11 +50,35 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
list(Db) ->
- {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
- Indexes.
+ DDocs = couch_views_ddoc:get_mango_list(Db),
+ DbName = fabric2_db:name(Db),
+ Indexes = lists:foldl(fun(DDoc, Acc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+
+ case proplists:get_value(<<"language">>, Props) == <<"query">> of
+ true ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+ IsInteractive = couch_views_ddoc:is_interactive(DDoc),
+ BuildState = couch_views_fdb:get_build_status(Db, Mrst),
+
+ Idxs = lists:map(fun(Idx) ->
+ Idx#idx{
+ build_status = BuildState,
+ interactive = IsInteractive
+ }
+ end, from_ddoc(Db, DDoc)),
+ Acc ++ Idxs;
+ false ->
+ Acc
+ end
+
+ end, [], DDocs),
+ Indexes ++ special(Db).
get_usable_indexes(Db, Selector, Opts) ->
@@ -62,13 +86,14 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
SortFields = get_sort_fields(Opts),
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
- case lists:filter(UsableFilter, UsableIndexes) of
+ case lists:filter(UsableFilter, UsableIndexes0) of
[] ->
mango_sort_error(Db, Opts);
UsableIndexes ->
@@ -162,16 +187,17 @@ delete(Filt, Db, Indexes, DelOpts) ->
end.
-from_ddoc(Db, {Props}) ->
+from_ddoc(Db, #doc{id = DDocId} = DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
DbName = db_to_name(Db),
- DDoc = proplists:get_value(<<"_id">>, Props),
+ DDocId = proplists:get_value(<<"_id">>, Props),
case proplists:get_value(<<"language">>, Props) of
<<"query">> -> ok;
_ ->
?MANGO_ERROR(invalid_query_ddoc_language)
end,
- IdxMods = case clouseau_rpc:connected() of
+ IdxMods = case is_text_service_available() of
true ->
[mango_idx_view, mango_idx_text];
false ->
@@ -181,7 +207,7 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc
+ ddoc = DDocId
}
end, Idxs).
@@ -192,7 +218,8 @@ special(Db) ->
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?INDEX_READY
},
% Add one for _update_seq
[AllDocs].
@@ -278,7 +305,7 @@ db_to_name(Name) when is_binary(Name) ->
db_to_name(Name) when is_list(Name) ->
iolist_to_binary(Name);
db_to_name(Db) ->
- couch_db:name(Db).
+ fabric2_db:name(Db).
get_idx_def(Opts) ->
@@ -293,7 +320,7 @@ get_idx_def(Opts) ->
get_idx_type(Opts) ->
case proplists:get_value(type, Opts) of
<<"json">> -> <<"json">>;
- <<"text">> -> case clouseau_rpc:connected() of
+ <<"text">> -> case is_text_service_available() of
true ->
<<"text">>;
false ->
@@ -306,6 +333,11 @@ get_idx_type(Opts) ->
end.
+is_text_service_available() ->
+ erlang:function_exported(clouseau_rpc, connected, 0) andalso
+ clouseau_rpc:connected().
+
+
get_idx_ddoc(Idx, Opts) ->
case proplists:get_value(ddoc, Opts) of
<<"_design/", _Rest/binary>> = Name ->
@@ -377,7 +409,8 @@ index(SelectorName, Selector) ->
<<"Selected">>,<<"json">>,
{[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
{SelectorName,{Selector}}]},
- [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
+ [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}],
+ <<"ready">>
}.
get_partial_filter_all_docs_test() ->
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..68e5aaa 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,7 @@
name,
type,
def,
- opts
+ opts,
+ build_status,
+ interactive
}).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..3548372 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -28,6 +28,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
validate(_) ->
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?INDEX_READY}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 2d784b6..84be418 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -54,7 +54,16 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
NewView = make_view(Idx),
Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
- {ok, DDoc#doc{body={Props1}}}.
+
+ {Opts0} = proplists:get_value(<<"options">>, Props1, {[]}),
+ Opts1 = case lists:keymember(<<"interactive">>, 1, Opts0) of
+ true -> Opts0;
+ false -> Opts0 ++ [{<<"interactive">>, true}]
+ end,
+ Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Opts1}}),
+
+ Props3 = [{<<"autoupdate">>, false}],
+ {ok, DDoc#doc{body={Props2 ++ Props3}}}.
remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -68,13 +77,15 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
if Views2 /= Views1 -> ok; true ->
?MANGO_ERROR({index_not_found, Idx#idx.name})
end,
- Props1 = case Views2 of
+ Props3 = case Views2 of
[] ->
- lists:keydelete(<<"views">>, 1, Props0);
+ Props1 = lists:keydelete(<<"views">>, 1, Props0),
+ Props2 = lists:keydelete(<<"options">>, 1, Props1),
+ lists:keydelete(<<"autoupdate">>, 1, Props2);
_ ->
lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}})
end,
- {ok, DDoc#doc{body={Props1}}}.
+ {ok, DDoc#doc{body={Props3}}}.
from_ddoc({Props}) ->
@@ -104,7 +115,8 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {def, {def_to_json(Idx#idx.def)}}
+ {def, {def_to_json(Idx#idx.def)}},
+ {build_status, Idx#idx.build_status}
]}.