You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/04/02 12:39:28 UTC
[couchdb] branch fdb-mango-indexes updated (e0aff0e -> 95477cc)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
omit e0aff0e remove defer and load_ddocs from mango_utils
omit 43221ff update mango tests to work with Mango on FDB
omit 19dea6d Update mango test creds to same as elixir tests
omit 2d1b346 Update Mango query to work with couch_views
omit 1a9195b Add mango indexing
omit c806526 Remove quorum stats
omit bb8dba2 Remove view_cb predicate push down
omit 94f86bf remove unneeded r/w parameter
omit 8499886 remove partition opts from mango
omit aab5a71 remove mango native proc
omit 0e86a6a Add couch_views_encoding max value
omit 836996d Add couch_views_updater interactive indexer
omit b73c5ab All couch_view queries to run across transactions
omit 174f382 Add couch_views_indexer build to creation versionstamp
omit cf0b2e0 add fabric2 after_doc_write plugin
omit 33a4c9f add include_docs option to fold_docs
omit e2b69e0 move all_doc view options to fabric2_util
omit d1dbad6 fix all_docs call to return row
add 24524a4 Switch erlfdb to the couchdb repo at tag v1.0.0
add 4707af4 Merge pull request #2743 from apache/switch-erlfdb-couch
add 85f81d8 Use `couch_rate` application for `couch_view`
add d520d73 Merge pull request #2662 from cloudant/couch_view-rate_limit
new a553bc2 fix all_docs call to return row
new 3179147 move all_doc view options to fabric2_util
new ec50197 add include_docs option to fold_docs
new bd192ea add fabric2 after_doc_write plugin
new 9dfe9e5 Add couch_views_indexer build to creation versionstamp
new 5000260 All couch_view queries to run across transactions
new 3c86190 Add couch_views_updater interactive indexer
new fd323c1 Add couch_views_encoding max value
new daecc5d remove mango native proc
new 0df103b remove partition opts from mango
new 6170372 remove unneeded r/w parameter
new 825aafc Remove view_cb predicate push down
new a2fc004 Remove quorum stats
new e438a36 Add mango indexing
new 5fc1900 Update Mango query to work with couch_views
new 370e490 Update mango test creds to same as elixir tests
new 36ed475 update mango tests to work with Mango on FDB
new 95477cc remove defer and load_ddocs from mango_utils
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e0aff0e)
\
N -- N -- N refs/heads/fdb-mango-indexes (95477cc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.credo.exs | 1 +
.gitignore | 1 +
Makefile | 1 +
mix.exs | 8 +-
mix.lock | 1 +
rebar.config.script | 9 +-
rel/files/eunit.ini | 3 +
rel/overlay/etc/default.ini | 4 +
src/couch_rate/README.md | 155 +++++++++
.../src/couch_rate.app.src} | 15 +-
src/couch_rate/src/couch_rate.erl | 318 +++++++++++++++++
.../src/couch_rate.hrl} | 22 +-
.../src/couch_rate_app.erl} | 6 +-
src/couch_rate/src/couch_rate_config.erl | 66 ++++
src/couch_rate/src/couch_rate_ets.erl | 119 +++++++
src/couch_rate/src/couch_rate_limiter.erl | 387 +++++++++++++++++++++
src/couch_rate/src/couch_rate_pd.erl | 90 +++++
.../src/couch_rate_sup.erl} | 13 +-
.../test/exunit/couch_rate_config_test.exs | 88 +++++
.../test/exunit/couch_rate_limiter_test.exs | 350 +++++++++++++++++++
.../couch_rate/test/exunit/test_helper.exs | 3 +
src/couch_views/README.md | 33 ++
src/couch_views/src/couch_views.app.src | 3 +-
src/couch_views/src/couch_views_indexer.erl | 61 ++--
src/couch_views/src/couch_views_jobs.erl | 7 +-
src/couch_views/test/couch_views_indexer_test.erl | 53 ++-
.../test/couch_views_trace_index_test.erl | 4 +-
src/mango/src/mango_idx_view.erl | 1 -
src/mango/test/13-users-db-find-test.py | 118 ++++---
29 files changed, 1809 insertions(+), 131 deletions(-)
create mode 100644 src/couch_rate/README.md
copy src/{couch_eval/src/couch_eval.app.src => couch_rate/src/couch_rate.app.src} (78%)
create mode 100644 src/couch_rate/src/couch_rate.erl
copy src/{couch/src/couch_rand.erl => couch_rate/src/couch_rate.hrl} (72%)
copy src/{ctrace/src/ctrace_app.erl => couch_rate/src/couch_rate_app.erl} (91%)
create mode 100644 src/couch_rate/src/couch_rate_config.erl
create mode 100644 src/couch_rate/src/couch_rate_ets.erl
create mode 100644 src/couch_rate/src/couch_rate_limiter.erl
create mode 100644 src/couch_rate/src/couch_rate_pd.erl
copy src/{ctrace/src/ctrace_sup.erl => couch_rate/src/couch_rate_sup.erl} (73%)
create mode 100644 src/couch_rate/test/exunit/couch_rate_config_test.exs
create mode 100644 src/couch_rate/test/exunit/couch_rate_limiter_test.exs
copy rel/files/vm.args => src/couch_rate/test/exunit/test_helper.exs (86%)
[couchdb] 14/18: Add mango indexing
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e438a36abeef4d3a6a288a236dfaeb1506e9b2cb
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 15:19:44 2020 +0200
Add mango indexing
This uses couch_views_updater to create mango indexes in the doc update
along with the couch_views_indexer to update the indexes in the
background up to the creation versionstamp.
---
rel/overlay/etc/default.ini | 1 +
src/couch_eval/src/couch_eval.erl | 3 +
src/mango/src/mango_eval.erl | 115 ++++++++++++++++++++++++++++++++++++
src/mango/src/mango_idx.erl | 57 ++++++++++++++----
src/mango/src/mango_idx.hrl | 4 +-
src/mango/src/mango_idx_special.erl | 4 +-
src/mango/src/mango_idx_view.erl | 22 +++++--
7 files changed, 187 insertions(+), 19 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index fd0aa77..d2a2c72 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -355,6 +355,7 @@ os_process_limit = 100
; beahvior for executing provided code in design
; documents.
javascript = couch_js
+query = mango_eval
[mango]
; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl
index 23ca263..3541a5b 100644
--- a/src/couch_eval/src/couch_eval.erl
+++ b/src/couch_eval/src/couch_eval.erl
@@ -75,6 +75,9 @@ acquire_map_context(DbName, DDocId, Language, Sig, Lib, MapFuns) ->
-spec release_map_context(context()) -> ok | {error, any()}.
+release_map_context(nil) ->
+ ok;
+
release_map_context({ApiMod, Ctx}) ->
ApiMod:release_map_context(Ctx).
diff --git a/src/mango/src/mango_eval.erl b/src/mango/src/mango_eval.erl
new file mode 100644
index 0000000..59d784b
--- /dev/null
+++ b/src/mango/src/mango_eval.erl
@@ -0,0 +1,115 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_eval).
+-behavior(couch_eval).
+
+
+-export([
+ acquire_map_context/1,
+ release_map_context/1,
+ map_docs/2
+]).
+
+
+-export([
+ index_doc/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_idx.hrl").
+
+
+acquire_map_context(Opts) ->
+ #{
+ db_name := DbName,
+ ddoc_id := DDocId,
+ map_funs := MapFuns
+ } = Opts,
+ Indexes = lists:map(fun (Def) ->
+ #idx{
+ type = <<"json">>,
+ dbname = DbName,
+ ddoc = DDocId,
+ def = Def
+ }
+ end, MapFuns),
+ {ok, Indexes}.
+
+
+release_map_context(_) ->
+ ok.
+
+
+map_docs(Indexes, Docs) ->
+ {ok, lists:map(fun(Doc) ->
+ Json = couch_doc:to_json_obj(Doc, []),
+ Results = index_doc(Indexes, Json),
+ {Doc#doc.id, Results}
+ end, Docs)}.
+
+
+index_doc(Indexes, Doc) ->
+ lists:map(fun(Idx) ->
+ {IdxDef} = mango_idx:def(Idx),
+ Results = get_index_entries(IdxDef, Doc),
+ case lists:member(not_found, Results) of
+ true ->
+ [];
+ false ->
+ [{Results, null}]
+ end
+ end, Indexes).
+
+
+get_index_entries(IdxDef, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxDef),
+ Selector = get_index_partial_filter_selector(IdxDef),
+ case should_index(Selector, Doc) of
+ false ->
+ [not_found];
+ true ->
+ get_index_values(Fields, Doc)
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxDef) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxDef, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the
+ % partial_filter_selector set as selector
+ couch_util:get_value(<<"selector">>, IdxDef, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index a26a685..ba9f68f 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -50,11 +50,35 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
list(Db) ->
- {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
- Indexes.
+ DDocs = couch_views_ddoc:get_mango_list(Db),
+ DbName = fabric2_db:name(Db),
+ Indexes = lists:foldl(fun(DDoc, Acc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+
+ case proplists:get_value(<<"language">>, Props) == <<"query">> of
+ true ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+ IsInteractive = couch_views_ddoc:is_interactive(DDoc),
+ BuildState = couch_views_fdb:get_build_status(Db, Mrst),
+
+ Idxs = lists:map(fun(Idx) ->
+ Idx#idx{
+ build_status = BuildState,
+ interactive = IsInteractive
+ }
+ end, from_ddoc(Db, DDoc)),
+ Acc ++ Idxs;
+ false ->
+ Acc
+ end
+
+ end, [], DDocs),
+ Indexes ++ special(Db).
get_usable_indexes(Db, Selector, Opts) ->
@@ -62,13 +86,14 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
SortFields = get_sort_fields(Opts),
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
- case lists:filter(UsableFilter, UsableIndexes) of
+ case lists:filter(UsableFilter, UsableIndexes0) of
[] ->
mango_sort_error(Db, Opts);
UsableIndexes ->
@@ -162,16 +187,17 @@ delete(Filt, Db, Indexes, DelOpts) ->
end.
-from_ddoc(Db, {Props}) ->
+from_ddoc(Db, #doc{id = DDocId} = DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
DbName = db_to_name(Db),
- DDoc = proplists:get_value(<<"_id">>, Props),
+ DDocId = proplists:get_value(<<"_id">>, Props),
case proplists:get_value(<<"language">>, Props) of
<<"query">> -> ok;
_ ->
?MANGO_ERROR(invalid_query_ddoc_language)
end,
- IdxMods = case clouseau_rpc:connected() of
+ IdxMods = case is_text_service_available() of
true ->
[mango_idx_view, mango_idx_text];
false ->
@@ -181,7 +207,7 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc
+ ddoc = DDocId
}
end, Idxs).
@@ -192,7 +218,8 @@ special(Db) ->
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?INDEX_READY
},
% Add one for _update_seq
[AllDocs].
@@ -278,7 +305,7 @@ db_to_name(Name) when is_binary(Name) ->
db_to_name(Name) when is_list(Name) ->
iolist_to_binary(Name);
db_to_name(Db) ->
- couch_db:name(Db).
+ fabric2_db:name(Db).
get_idx_def(Opts) ->
@@ -293,7 +320,7 @@ get_idx_def(Opts) ->
get_idx_type(Opts) ->
case proplists:get_value(type, Opts) of
<<"json">> -> <<"json">>;
- <<"text">> -> case clouseau_rpc:connected() of
+ <<"text">> -> case is_text_service_available() of
true ->
<<"text">>;
false ->
@@ -306,6 +333,11 @@ get_idx_type(Opts) ->
end.
+is_text_service_available() ->
+ erlang:function_exported(clouseau_rpc, connected, 0) andalso
+ clouseau_rpc:connected().
+
+
get_idx_ddoc(Idx, Opts) ->
case proplists:get_value(ddoc, Opts) of
<<"_design/", _Rest/binary>> = Name ->
@@ -377,7 +409,8 @@ index(SelectorName, Selector) ->
<<"Selected">>,<<"json">>,
{[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
{SelectorName,{Selector}}]},
- [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
+ [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}],
+ <<"ready">>
}.
get_partial_filter_all_docs_test() ->
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..68e5aaa 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,7 @@
name,
type,
def,
- opts
+ opts,
+ build_status,
+ interactive
}).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..3548372 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -28,6 +28,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
validate(_) ->
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?INDEX_READY}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 2d784b6..84be418 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -54,7 +54,16 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
NewView = make_view(Idx),
Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
- {ok, DDoc#doc{body={Props1}}}.
+
+ {Opts0} = proplists:get_value(<<"options">>, Props1, {[]}),
+ Opts1 = case lists:keymember(<<"interactive">>, 1, Opts0) of
+ true -> Opts0;
+ false -> Opts0 ++ [{<<"interactive">>, true}]
+ end,
+ Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Opts1}}),
+
+ Props3 = [{<<"autoupdate">>, false}],
+ {ok, DDoc#doc{body={Props2 ++ Props3}}}.
remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -68,13 +77,15 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
if Views2 /= Views1 -> ok; true ->
?MANGO_ERROR({index_not_found, Idx#idx.name})
end,
- Props1 = case Views2 of
+ Props3 = case Views2 of
[] ->
- lists:keydelete(<<"views">>, 1, Props0);
+ Props1 = lists:keydelete(<<"views">>, 1, Props0),
+ Props2 = lists:keydelete(<<"options">>, 1, Props1),
+ lists:keydelete(<<"autoupdate">>, 1, Props2);
_ ->
lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}})
end,
- {ok, DDoc#doc{body={Props1}}}.
+ {ok, DDoc#doc{body={Props3}}}.
from_ddoc({Props}) ->
@@ -104,7 +115,8 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {def, {def_to_json(Idx#idx.def)}}
+ {def, {def_to_json(Idx#idx.def)}},
+ {build_status, Idx#idx.build_status}
]}.
[couchdb] 07/18: Add couch_views_updater interactive indexer
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3c861907d09325c1736435073da3fefdee977d74
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 25 15:38:53 2020 +0200
Add couch_views_updater interactive indexer
This adds the ability for couch_views to index an index in the docs
update transaction. This only happens if a design doc has the
field <<"interactive">> = true.
---
rel/apps/couch_epi.config | 1 +
src/couch_views/src/couch_views.app.src | 1 +
src/couch_views/src/couch_views.erl | 16 +-
src/couch_views/src/couch_views_ddoc.erl | 42 ++++
src/couch_views/src/couch_views_epi.erl | 58 ++++++
.../couch_views/src/couch_views_fabric2_plugin.erl | 23 ++-
src/couch_views/src/couch_views_sup.erl | 2 +-
src/couch_views/src/couch_views_updater.erl | 101 +++++++++
src/couch_views/test/couch_views_updater_test.erl | 230 +++++++++++++++++++++
9 files changed, 458 insertions(+), 16 deletions(-)
diff --git a/rel/apps/couch_epi.config b/rel/apps/couch_epi.config
index 0f3d2da..d371163 100644
--- a/rel/apps/couch_epi.config
+++ b/rel/apps/couch_epi.config
@@ -15,6 +15,7 @@
fabric2_epi,
chttpd_epi,
couch_index_epi,
+ couch_views_epi,
dreyfus_epi,
global_changes_epi,
mango_epi,
diff --git a/src/couch_views/src/couch_views.app.src b/src/couch_views/src/couch_views.app.src
index b704c97..cb8285a 100644
--- a/src/couch_views/src/couch_views.app.src
+++ b/src/couch_views/src/couch_views.app.src
@@ -22,6 +22,7 @@
kernel,
stdlib,
erlfdb,
+ couch_epi,
couch_log,
config,
couch_stats,
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 2268052..2acba00 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -37,6 +37,7 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
end,
DbName = fabric2_db:name(Db),
+ IsInteractive = couch_views_ddoc:is_interactive(DDoc),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
#mrst{
@@ -54,7 +55,7 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
- ok = maybe_update_view(TxDb, Mrst, Args3),
+ ok = maybe_update_view(TxDb, Mrst, IsInteractive, Args3),
read_view(TxDb, Mrst, ViewName, Callback, Acc0, Args3)
end)
catch throw:{build_view, WaitSeq} ->
@@ -127,13 +128,20 @@ read_view(Db, Mrst, ViewName, Callback, Acc0, Args) ->
end).
-maybe_update_view(_Db, _Mrst, #mrargs{update = false}) ->
+maybe_update_view(_Db, _Mrst, _, #mrargs{update = false}) ->
ok;
-maybe_update_view(_Db, _Mrst, #mrargs{update = lazy}) ->
+maybe_update_view(_Db, _Mrst, _, #mrargs{update = lazy}) ->
ok;
-maybe_update_view(TxDb, Mrst, _Args) ->
+maybe_update_view(TxDb, Mrst, true, _Args) ->
+ BuildState = couch_views_fdb:get_build_status(TxDb, Mrst),
+ if BuildState == ?INDEX_READY -> ok; true ->
+ VS = couch_views_fdb:get_creation_vs(TxDb, Mrst),
+ throw({build_view, fabric2_fdb:vs_to_seq(VS)})
+ end;
+
+maybe_update_view(TxDb, Mrst, false, _Args) ->
DbSeq = fabric2_db:get_update_seq(TxDb),
ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
case DbSeq == ViewSeq of
diff --git a/src/couch_views/src/couch_views_ddoc.erl b/src/couch_views/src/couch_views_ddoc.erl
new file mode 100644
index 0000000..fae4a34
--- /dev/null
+++ b/src/couch_views/src/couch_views_ddoc.erl
@@ -0,0 +1,42 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_views_ddoc).
+
+
+-export([
+ get_interactive_list/1,
+ get_mango_list/1,
+ is_interactive/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+% TODO: build a ddoc cache that checks the md_version
+get_interactive_list(Db) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ lists:filter(fun is_interactive/1, DDocs).
+
+
+get_mango_list(Db) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ lists:filter(fun (DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+ fabric2_util:get_value(<<"language">>, Props) == <<"query">>
+ end, DDocs).
+
+
+is_interactive(#doc{} = DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+ {Opts} = fabric2_util:get_value(<<"options">>, Props, {[]}),
+ fabric2_util:get_value(<<"interactive">>, Opts, false).
diff --git a/src/couch_views/src/couch_views_epi.erl b/src/couch_views/src/couch_views_epi.erl
new file mode 100644
index 0000000..6d39d9a
--- /dev/null
+++ b/src/couch_views/src/couch_views_epi.erl
@@ -0,0 +1,58 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_views_epi).
+
+
+-behaviour(couch_epi_plugin).
+
+
+-export([
+ app/0,
+ providers/0,
+ services/0,
+ data_subscriptions/0,
+ data_providers/0,
+ processes/0,
+ notify/3
+]).
+
+
+app() ->
+ couch_views.
+
+
+providers() ->
+ [
+ {fabric2_db, couch_views_fabric2_plugin}
+ ].
+
+
+services() ->
+ [].
+
+
+data_subscriptions() ->
+ [].
+
+
+data_providers() ->
+ [].
+
+
+processes() ->
+ [].
+
+
+notify(_Key, _Old, _New) ->
+ ok.
diff --git a/rel/apps/couch_epi.config b/src/couch_views/src/couch_views_fabric2_plugin.erl
similarity index 66%
copy from rel/apps/couch_epi.config
copy to src/couch_views/src/couch_views_fabric2_plugin.erl
index 0f3d2da..cae0e1f 100644
--- a/rel/apps/couch_epi.config
+++ b/src/couch_views/src/couch_views_fabric2_plugin.erl
@@ -10,14 +10,15 @@
% License for the specific language governing permissions and limitations under
% the License.
-{plugins, [
- couch_db_epi,
- fabric2_epi,
- chttpd_epi,
- couch_index_epi,
- dreyfus_epi,
- global_changes_epi,
- mango_epi,
- mem3_epi,
- setup_epi
-]}.
+
+-module(couch_views_fabric2_plugin).
+
+
+-export([
+ after_doc_write/6
+]).
+
+
+after_doc_write(Db, Doc, NewWinner, OldWinner, NewRevId, Seq)->
+ couch_views_updater:index(Db, Doc, NewWinner, OldWinner, NewRevId, Seq),
+ [Db, Doc, NewWinner, OldWinner, NewRevId, Seq].
diff --git a/src/couch_views/src/couch_views_sup.erl b/src/couch_views/src/couch_views_sup.erl
index 2a40f0a..9453189 100644
--- a/src/couch_views/src/couch_views_sup.erl
+++ b/src/couch_views/src/couch_views_sup.erl
@@ -42,7 +42,7 @@ init(normal) ->
id => couch_views_server,
start => {couch_views_server, start_link, []}
}
- ],
+ ] ++ couch_epi:register_service(couch_views_epi, []),
{ok, {flags(), Children}};
init(builds_disabled) ->
diff --git a/src/couch_views/src/couch_views_updater.erl b/src/couch_views/src/couch_views_updater.erl
new file mode 100644
index 0000000..f405123
--- /dev/null
+++ b/src/couch_views/src/couch_views_updater.erl
@@ -0,0 +1,101 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_views_updater).
+
+-export([
+ index/6
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
+% If the doc revision doesn't not match the NewRevId passed here we can ignore
+% the document since it is then a conflict document and it doesn't need
+% to be indexed.
+index(Db, #doc{id = Id, revs = Revs} = Doc, _NewWinner, _OldWinner, NewRevId,
+ Seq) ->
+ try
+ {Depth, [FirstRev | _]} = Revs,
+ DocRev = {Depth, FirstRev},
+ if DocRev /= NewRevId -> ok; true ->
+ index_int(Db, Doc, Seq)
+ end
+ catch
+ Error:Reason ->
+ DbName = fabric2_db:name(Db),
+ couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+ [DbName, Id, Error, Reason])
+ end.
+
+
+% Check if design doc is an interactive index and kick off background worker
+% to build the new index up to the creation_vs
+index_int(Db, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>,
+ deleted = false} = DDoc, Seq) ->
+ DbName = fabric2_db:name(Db),
+
+ case couch_views_ddoc:is_interactive(DDoc) of
+ true ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+ case couch_views_fdb:get_creation_vs(Db, Mrst) of
+ not_found ->
+ couch_views_fdb:new_interactive_index(Db, Mrst, Seq),
+ {ok, _} = couch_views_jobs:build_view_async(Db, Mrst);
+ _ ->
+ ok
+ end;
+ false ->
+ ok
+ end,
+ write_doc(Db, DDoc);
+
+
+index_int(Db, #doc{} = Doc, _Seq) ->
+ write_doc(Db, Doc).
+
+
+write_doc(Db, #doc{deleted = Deleted} = Doc) ->
+ DbName = fabric2_db:name(Db),
+ DDocs = couch_views_ddoc:get_interactive_list(Db),
+
+ Result0 = [#{
+ id => Doc#doc.id,
+ results => [],
+ deleted => Deleted,
+ doc => Doc
+ }],
+
+ %% Interactive updates do not update the views update_seq
+ State = #{
+ last_seq => false
+ },
+
+ lists:foreach(fun(DDoc) ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+ case should_index_doc(Doc, Mrst) of
+ true ->
+ {Mrst1, Result1} = couch_views_indexer:map_docs(Mrst, Result0),
+ couch_views_indexer:write_docs(Db, Mrst1, Result1, State),
+ couch_eval:release_map_context(Mrst1#mrst.qserver);
+ false ->
+ ok
+ end
+ end, DDocs).
+
+
+should_index_doc(<<?DESIGN_DOC_PREFIX, _/binary>>, Mrst) ->
+ lists:keymember(<<"include_design">>, 1, Mrst#mrst.design_opts);
+
+should_index_doc(_, _) ->
+ true.
diff --git a/src/couch_views/test/couch_views_updater_test.erl b/src/couch_views/test/couch_views_updater_test.erl
new file mode 100644
index 0000000..e456225
--- /dev/null
+++ b/src/couch_views/test/couch_views_updater_test.erl
@@ -0,0 +1,230 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_updater_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+-include_lib("mango/src/mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
+
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [
+ ?TDEF_FE(index_docs),
+ ?TDEF_FE(update_doc),
+ ?TDEF_FE(delete_doc),
+ ?TDEF_FE(includes_design_docs)
+ ]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views,
+ mango
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_idx_ddoc(),
+ fabric2_db:update_docs(Db, [DDoc]),
+ % make sure the index is built for the first time so the background
+ % indexer doesn't build the index
+ wait_while_ddoc_builds(Db),
+
+ Docs = make_docs(3),
+ fabric2_db:update_docs(Db, Docs),
+ {Db, DDoc}.
+
+
+foreach_teardown({Db, _}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+update_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"2">>}, {value, 4}]
+ ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+includes_design_docs({Db, _}) ->
+ DDoc = create_idx_include_ddocs(),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ IndexDDoc0 = create_idx_ddoc(),
+ IndexDDoc = IndexDDoc0#doc{
+ id = <<"_design/to_be_indexed">>
+ },
+
+ fabric2_db:update_docs(Db, [IndexDDoc]),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"_design/ddoc_that_indexes_ddocs">>}, {value, 1}],
+ [{id, <<"_design/to_be_indexed">>}, {value, 1}]
+ ], Docs).
+
+
+run_query(Db, DDoc) ->
+ Args = #mrargs{
+ view_type = map,
+ reduce = false,
+ include_docs = true,
+ update = false
+ },
+ CB = fun query_cb/2,
+ {ok, Acc} = couch_views:query(Db, DDoc, <<"idx_01">>, CB, [], Args),
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props, 1)}
+ ]
+
+ end, Acc).
+
+
+create_idx_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc1">>},
+ {<<"language">>, <<"query">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, {[
+ {<<"fields">>, {[{<<"value">>, <<"asc">>}]}}
+ ]}},
+ {<<"reduce">>, <<"_count">>},
+ {<<"options">>, {[
+ {<<"def">>,
+ {[{<<"fields">>,
+ {[{<<"value">>, <<"asc">>}]}}]}}
+ ]}}
+ ]}}
+ ]}
+ },
+ {<<"autoupdate">>, false},
+ {<<"options">>, {[{<<"interactive">>, true}]}}
+ ]}).
+
+
+create_idx_include_ddocs() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc_that_indexes_ddocs">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, <<
+ "function(doc) {"
+ "if (doc.language) {"
+ "emit(doc.language, 1);"
+ "}"
+ "}">>}
+ ]}}
+ ]}},
+ {<<"autoupdate">>, false},
+ {<<"options">>, {[
+ {<<"include_design">>, true},
+ {<<"interactive">>, true}
+ ]}}
+ ]}).
+
+
+wait_while_ddoc_builds(Db) ->
+ Fun = fun () ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Ready = lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?INDEX_READY
+ end, mango_idx:list(TxDb)),
+
+ if length(Ready) > 1 -> ok; true ->
+ wait
+ end
+ end)
+ end,
+ test_util:wait(Fun).
+
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({row, Props}, Acc) ->
+ Doc = couch_util:get_value(doc, Props),
+ {ok, Acc ++ [Doc]};
+
+query_cb(_, Acc) ->
+ {ok, Acc}.
+
[couchdb] 15/18: Update Mango query to work with couch_views
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5fc19004a42b9ba2ae55cc17fa1b7840374dff47
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:21:22 2020 +0200
Update Mango query to work with couch_views
---
src/couch_views/test/couch_views_map_test.erl | 41 +++++++++-
src/mango/src/mango_crud.erl | 24 ++----
src/mango/src/mango_cursor.erl | 4 +-
src/mango/src/mango_cursor_view.erl | 70 +++++------------
src/mango/src/mango_httpd.erl | 105 ++++++++++++++------------
src/mango/src/mango_idx.erl | 13 +++-
src/mango/src/mango_idx_view.erl | 5 +-
src/mango/src/mango_idx_view.hrl | 13 ----
src/mango/src/mango_util.erl | 11 +--
9 files changed, 139 insertions(+), 147 deletions(-)
diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl
index f8ba183..7d1e94b 100644
--- a/src/couch_views/test/couch_views_map_test.erl
+++ b/src/couch_views/test/couch_views_map_test.erl
@@ -14,6 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
-define(TDEF(A), {atom_to_list(A), fun A/0}).
@@ -56,7 +57,8 @@ map_views_test_() ->
?TDEF(should_map_duplicate_keys),
?TDEF(should_map_with_doc_emit),
?TDEF(should_map_update_is_false),
- ?TDEF(should_map_update_is_lazy)
+ ?TDEF(should_map_update_is_lazy),
+ ?TDEF(should_map_wait_for_interactive)
% fun should_give_ext_size_seq_indexed_test/1
]
}
@@ -419,6 +421,25 @@ should_map_update_is_lazy() ->
?assertEqual(Expect, Result2).
+should_map_wait_for_interactive() ->
+ DbName = ?tempdb(),
+ {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_interactive_ddoc(),
+ Docs = make_docs(101),
+
+ fabric2_db:update_docs(Db, Docs),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Result = couch_views:query(Db, DDoc, <<"idx_01">>, fun default_cb/2, [],
+ #{limit => 3}),
+ ?assertEqual({ok, [
+ {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}
+ ]}, Result).
+
+
% should_give_ext_size_seq_indexed_test(Db) ->
% DDoc = couch_doc:from_json_obj({[
% {<<"_id">>, <<"_design/seqdoc">>},
@@ -510,6 +531,24 @@ create_ddoc() ->
]}}
]}).
+create_interactive_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc_interactive">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, <<
+ "function(doc) {"
+ "if (doc.val) {"
+ "emit(doc.val, doc.val);"
+ "}"
+ "}">>}
+ ]}}
+ ]}},
+ {<<"autoupdate">>, false},
+ {<<"interactive">>, true}
+ ]}).
+
make_docs(Count) ->
[doc(I) || I <- lists:seq(1, Count)].
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 42717ff..66cef65 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -33,9 +33,8 @@ insert(Db, #doc{}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
-insert(Db, Docs, Opts0) when is_list(Docs) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- case fabric:update_docs(Db, Docs, Opts1) of
+insert(Db, Docs, Opts) when is_list(Docs) ->
+ case fabric2_db:update_docs(Db, Docs, Opts) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -45,9 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
end.
-find(Db, Selector, Callback, UserAcc, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+find(Db, Selector, Callback, UserAcc, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -97,21 +95,11 @@ delete(Db, Selector, Options) ->
end.
-explain(Db, Selector, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+explain(Db, Selector, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:explain(Cursor).
-maybe_add_user_ctx(Db, Opts) ->
- case lists:keyfind(user_ctx, 1, Opts) of
- {user_ctx, _} ->
- Opts;
- false ->
- [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
- end.
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index db4e981..63b449c 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -48,7 +48,9 @@
create(Db, Selector0, Opts) ->
Selector = mango_selector:normalize(Selector0),
- UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+ UsableIndexes = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ mango_idx:get_usable_indexes(TxDb, Selector, Opts)
+ end),
case mango_cursor:maybe_filter_indexes_by_ddoc(UsableIndexes, Opts) of
[] ->
% use_index doesn't match a valid index - fall back to a valid one
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index b88f6ea..4960fa1 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -31,9 +31,7 @@
-include_lib("fabric/include/fabric.hrl").
-include("mango_cursor.hrl").
--include("mango_idx_view.hrl").
--define(HEARTBEAT_INTERVAL_IN_USEC, 4000000).
create(Db, Indexes, Selector, Opts) ->
FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -91,7 +89,8 @@ maybe_replace_max_json(?MAX_STR) ->
<<"<MAX>">>;
maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
- H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+ MAX_VAL = couch_views_encoding:max(),
+ H1 = if H == MAX_VAL -> <<"<MAX>">>;
true -> H
end,
[H1 | maybe_replace_max_json(T)];
@@ -100,7 +99,7 @@ maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
+base_args(#cursor{index = Idx} = Cursor) ->
{StartKey, EndKey} = case Cursor#cursor.ranges of
[empty] ->
{null, null};
@@ -132,18 +131,19 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
Args = mango_json_bookmark:update_args(Bookmark, Args0),
- UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
- DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
all_docs ->
CB = fun ?MODULE:handle_all_docs_message/2,
- fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+ AllDocOpts = fabric2_util:all_docs_view_opts(Args)
+ ++ [{restart_tx, true}],
+ fabric2_db:fold_docs(Db, CB, Cursor, AllDocOpts);
_ ->
CB = fun ?MODULE:handle_message/2,
% Normal view
- DDoc = ddocid(Idx),
+ DDocId = mango_idx:ddoc(Idx),
+ {ok, DDoc} = fabric2_db:open_doc(Db, DDocId),
Name = mango_idx:name(Idx),
- fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args)
+ couch_views:query(Db, DDoc, Name, CB, Cursor, Args)
end,
case Result of
{ok, LastCursor} ->
@@ -227,7 +227,7 @@ choose_best_index(_DbName, IndexRanges) ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
- case doc_member(Cursor, Props) of
+ case match_doc(Cursor, Props) of
{ok, Doc, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
execution_stats = Stats
@@ -280,15 +280,6 @@ handle_doc(C, _Doc) ->
{stop, C}.
-ddocid(Idx) ->
- case mango_idx:ddoc(Idx) of
- <<"_design/", Rest/binary>> ->
- Rest;
- Else ->
- Else
- end.
-
-
apply_opts([], Args) ->
Args;
apply_opts([{conflicts, true} | Rest], Args) ->
@@ -340,41 +331,18 @@ apply_opts([{_, _} | Rest], Args) ->
apply_opts(Rest, Args).
-doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
- Opts = Cursor#cursor.opts,
- ExecutionStats = Cursor#cursor.execution_stats,
- Selector = Cursor#cursor.selector,
- case couch_util:get_value(doc, RowProps) of
- {DocProps} ->
- % only matching documents are returned; the selector
- % is evaluated at the shard level in view_cb({row, Row},
- {ok, {DocProps}, {execution_stats, ExecutionStats}};
- undefined ->
- % an undefined doc was returned, indicating we should
- % perform a quorum fetch
- ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
- couch_stats:increment_counter([mango, quorum_docs_examined]),
- Id = couch_util:get_value(id, RowProps),
- case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
- {ok, #doc{}=DocProps} ->
- Doc = couch_doc:to_json_obj(DocProps, []),
- match_doc(Selector, Doc, ExecutionStats1);
- Else ->
- Else
- end;
- _ ->
- % no doc, no match
- {no_match, null, {execution_stats, ExecutionStats}}
- end.
-
-
-match_doc(Selector, Doc, ExecutionStats) ->
+match_doc(Cursor, RowProps) ->
+ #cursor{
+ execution_stats = Stats0,
+ selector = Selector
+ } = Cursor,
+ Stats1 = mango_execution_stats:incr_docs_examined(Stats0, 1),
+ Doc = couch_util:get_value(doc, RowProps),
case mango_selector:match(Selector, Doc) of
true ->
- {ok, Doc, {execution_stats, ExecutionStats}};
+ {ok, Doc, {execution_stats, Stats1}};
false ->
- {no_match, Doc, {execution_stats, ExecutionStats}}
+ {no_match, Doc, {execution_stats, Stats1}}
end.
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 1054c74..94aa866 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,9 +32,8 @@
threshold = 1490
}).
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
try
- Db = set_user_ctx(Req, Db0),
handle_req_int(Req, Db)
catch
throw:{mango_error, Module, Reason} ->
@@ -61,7 +60,9 @@ handle_req_int(_, _) ->
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
- Idxs = lists:sort(mango_idx:list(Db)),
+ Idxs = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ lists:sort(mango_idx:list(TxDb))
+ end),
JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
TotalRows = length(JsonIdxs0),
Limit = case couch_util:get_value(limit, Params, TotalRows) of
@@ -87,25 +88,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, Idx0} = mango_idx:new(Db, Opts),
{ok, Idx} = mango_idx:validate_new(Idx0, Db),
DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body],
- {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts),
- Id = Idx#idx.ddoc,
- Name = Idx#idx.name,
- Status = case mango_idx:add(DDoc, Idx) of
- {ok, DDoc} ->
- <<"exists">>;
- {ok, NewDDoc} ->
- case mango_crud:insert(Db, NewDDoc, Opts) of
- {ok, [{RespProps}]} ->
- case lists:keyfind(error, 1, RespProps) of
- {error, Reason} ->
- ?MANGO_ERROR({error_saving_ddoc, Reason});
- _ ->
- <<"created">>
- end;
- _ ->
- ?MANGO_ERROR(error_saving_ddoc)
- end
- end,
+ Id = mango_idx:ddoc(Idx),
+ Name = mango_idx:name(Idx),
+ Status = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ {ok, DDoc} = mango_util:load_ddoc(TxDb, Id, DbOpts),
+ case mango_idx:add(DDoc, Idx) of
+ {ok, DDoc} ->
+ <<"exists">>;
+ {ok, NewDDoc} ->
+ case mango_crud:insert(TxDb, NewDDoc, Opts) of
+ {ok, [{RespProps}]} ->
+ case lists:keyfind(error, 1, RespProps) of
+ {error, Reason} ->
+ ?MANGO_ERROR({error_saving_ddoc, Reason});
+ _ ->
+ <<"created">>
+ end;
+ _ ->
+ ?MANGO_ERROR(error_saving_ddoc)
+ end
+ end
+ end),
chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) ->
@@ -118,19 +121,21 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
<<"_bulk_delete">>]}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
- Idxs = mango_idx:list(Db),
- DDocs = get_bulk_delete_ddocs(Opts),
- {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
- Id = {<<"id">>, DDocId},
- case mango_idx:delete(Filt, Db, Idxs, Opts) of
- {ok, true} ->
- {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
- {error, Error} ->
- {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
- end
- end, {[], []}, DDocs),
+ {Success, Fail} = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocs = get_bulk_delete_ddocs(Opts),
+ lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
+ Id = {<<"id">>, DDocId},
+ case mango_idx:delete(Filt, TxDb, Idxs, Opts) of
+ {ok, true} ->
+ {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
+ {error, Error} ->
+ {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
+ end
+ end, {[], []}, DDocs)
+ end),
chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]});
handle_index_req(#httpd{path_parts=[_, <<"_index">>,
@@ -144,15 +149,18 @@ handle_index_req(#httpd{method='DELETE',
handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
- Idxs = mango_idx:list(Db),
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) ->
- IsDDoc = mango_idx:ddoc(Idx) == DDocId,
- IsType = mango_idx:type(Idx) == Type,
- IsName = mango_idx:name(Idx) == Name,
- IsDDoc andalso IsType andalso IsName
- end,
- case mango_idx:delete(Filt, Db, Idxs, []) of
+ Result = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) ->
+ IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+ IsType = mango_idx:type(Idx) == Type,
+ IsName = mango_idx:name(Idx) == Name,
+ IsDDoc andalso IsType andalso IsName
+ end,
+ mango_idx:delete(Filt, TxDb, Idxs, [])
+ end),
+ case Result of
{ok, true} ->
chttpd:send_json(Req, {[{ok, true}]});
{error, not_found} ->
@@ -170,7 +178,9 @@ handle_explain_req(#httpd{method='POST'}=Req, Db) ->
Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
- Resp = mango_crud:explain(Db, Sel, Opts),
+ Resp = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ mango_crud:explain(TxDb, Sel, Opts)
+ end),
chttpd:send_json(Req, Resp);
handle_explain_req(Req, _Db) ->
@@ -195,11 +205,6 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
- {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
- NewDb.
-
-
get_bulk_delete_ddocs(Opts) ->
case lists:keyfind(docids, 1, Opts) of
{docids, DDocs} when is_list(DDocs) ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index ba9f68f..99a3588 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -86,7 +86,7 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
- BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
+ BuiltIndexes = remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
@@ -399,6 +399,17 @@ get_legacy_selector(Def) ->
Selector -> Selector
end.
+% remove any interactive indexes that are not built. If an index is not
+% interactive than we do not remove it as it will be built when queried
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun(Idx) ->
+ case Idx#idx.interactive of
+ true -> Idx#idx.build_status == ?INDEX_READY;
+ _ -> true
+ end
+ end, Indexes).
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 84be418..f80cc21 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,7 +34,6 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
--include("mango_idx_view.hrl").
validate_new(#idx{}=Idx, _Db) ->
@@ -183,11 +182,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
false ->
[Key | end_key(Rest)]
end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
deleted file mode 100644
index 0d213e5..0000000
--- a/src/mango/src/mango_idx_view.hrl
+++ /dev/null
@@ -1,13 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 0d31f15..18a6439 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,7 @@ open_doc(Db, DocId) ->
open_doc(Db, DocId, Options) ->
- case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
- {ok, Doc} ->
- {ok, Doc};
- {not_found, _} ->
- not_found;
- _ ->
- ?MANGO_ERROR({error_loading_doc, DocId})
- end.
+ fabric2_db:open_doc(Db, DocId, Options).
open_ddocs(Db) ->
@@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
case open_doc(Db, DDocId, DbOpts) of
{ok, Doc} ->
{ok, check_lang(Doc)};
- not_found ->
+ {not_found, missing} ->
Body = {[
{<<"language">>, <<"query">>}
]},
[couchdb] 10/18: remove partition opts from mango
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0df103b46807f95aebe0b0955115a19842678159
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 24 14:23:09 2020 +0200
remove partition opts from mango
---
src/mango/src/mango_cursor.erl | 1 -
src/mango/src/mango_cursor_text.erl | 9 ---
src/mango/src/mango_cursor_view.erl | 6 --
src/mango/src/mango_error.erl | 14 ----
src/mango/src/mango_httpd.erl | 21 +-----
src/mango/src/mango_idx.erl | 126 +++---------------------------------
src/mango/src/mango_idx.hrl | 1 -
src/mango/src/mango_idx_text.erl | 1 -
src/mango/src/mango_idx_view.erl | 1 -
src/mango/src/mango_opts.erl | 30 ---------
10 files changed, 10 insertions(+), 200 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index b1cb414..f16765b 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -72,7 +72,6 @@ explain(#cursor{}=Cursor) ->
{[
{dbname, mango_idx:dbname(Idx)},
{index, mango_idx:to_json(Idx)},
- {partitioned, mango_idx:partitioned(Idx)},
{selector, Selector},
{opts, {Opts}},
{limit, Limit},
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 43ef84e..ccf58ad 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -77,7 +77,6 @@ explain(Cursor) ->
} = Cursor,
[
{'query', mango_selector_text:convert(Selector)},
- {partition, get_partition(Opts, null)},
{sort, sort_query(Opts, Selector)}
].
@@ -95,7 +94,6 @@ execute(Cursor, UserFun, UserAcc) ->
Query = mango_selector_text:convert(Selector),
QueryArgs = #index_query_args{
q = Query,
- partition = get_partition(Opts, nil),
sort = sort_query(Opts, Selector),
raw_bookmark = true
},
@@ -250,13 +248,6 @@ sort_query(Opts, Selector) ->
end.
-get_partition(Opts, Default) ->
- case couch_util:get_value(partition, Opts) of
- <<>> -> Default;
- Else -> Else
- end.
-
-
get_bookmark(Opts) ->
case lists:keyfind(bookmark, 1, Opts) of
{_, BM} when is_list(BM), BM /= [] ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 240ef50..bced842 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -73,7 +73,6 @@ explain(Cursor) ->
{include_docs, Args#mrargs.include_docs},
{view_type, Args#mrargs.view_type},
{reduce, Args#mrargs.reduce},
- {partition, couch_mrview_util:get_extra(Args, partition, null)},
{start_key, maybe_replace_max_json(Args#mrargs.start_key)},
{end_key, maybe_replace_max_json(Args#mrargs.end_key)},
{direction, Args#mrargs.direction},
@@ -410,11 +409,6 @@ apply_opts([{update, false} | Rest], Args) ->
update = false
},
apply_opts(Rest, NewArgs);
-apply_opts([{partition, <<>>} | Rest], Args) ->
- apply_opts(Rest, Args);
-apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
- NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
- apply_opts(Rest, NewArgs);
apply_opts([{_, _} | Rest], Args) ->
% Ignore unknown options
apply_opts(Rest, Args).
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
index bb545ad..9ac8f63 100644
--- a/src/mango/src/mango_error.erl
+++ b/src/mango/src/mango_error.erl
@@ -28,13 +28,6 @@ info(mango_idx, {no_usable_index, missing_sort_index}) ->
<<"No index exists for this sort, "
"try indexing by the sort fields.">>
};
-info(mango_idx, {no_usable_index, missing_sort_index_partitioned}) ->
- {
- 400,
- <<"no_usable_index">>,
- <<"No partitioned index exists for this sort, "
- "try indexing by the sort fields.">>
- };
info(mango_idx, {no_usable_index, missing_sort_index_global}) ->
{
400,
@@ -118,13 +111,6 @@ info(mango_idx, {invalid_index_type, BadType}) ->
<<"invalid_index">>,
fmt("Invalid type for index: ~s", [BadType])
};
-info(mango_idx, {partitioned_option_mismatch, BadDDoc}) ->
- {
- 400,
- <<"invalid_partitioned_option">>,
- fmt("Requested partitioned option does not match existing value on"
- " design document ~s", [BadDDoc])
- };
info(mango_idx, invalid_query_ddoc_language) ->
{
400,
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 379d2e1..946d7e4 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -170,7 +170,7 @@ handle_index_req(#httpd{path_parts=[_, _, _DDocId0, _Type, _Name]}=Req, _Db) ->
handle_explain_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- Body = maybe_set_partition(Req),
+ Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
Resp = mango_crud:explain(Db, Sel, Opts),
@@ -182,7 +182,7 @@ handle_explain_req(Req, _Db) ->
handle_find_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- Body = maybe_set_partition(Req),
+ Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
{ok, Resp0} = start_find_resp(Req),
@@ -231,23 +231,6 @@ get_idx_del_opts(Req) ->
end.
-maybe_set_partition(Req) ->
- {Props} = chttpd:json_body_obj(Req),
- case chttpd:qs_value(Req, "partition", undefined) of
- undefined ->
- {Props};
- Partition ->
- case couch_util:get_value(<<"partition">>, Props) of
- undefined ->
- {[{<<"partition">>, ?l2b(Partition)} | Props]};
- Partition ->
- {Props};
- OtherPartition ->
- ?MANGO_ERROR({bad_partition, OtherPartition})
- end
- end.
-
-
convert_to_design_id(DDocId) ->
case DDocId of
<<"_design/", _/binary>> -> DDocId;
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5d06a8f..0f79bdb 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -33,7 +33,6 @@
name/1,
type/1,
def/1,
- partitioned/1,
opts/1,
columns/1,
is_usable/3,
@@ -64,13 +63,12 @@ get_usable_indexes(Db, Selector, Opts) ->
ExistingIndexes
),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
- UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts),
+ UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
SortFields = get_sort_fields(Opts),
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
- case lists:filter(UsableFilter, UsableIndexes1) of
+ case lists:filter(UsableFilter, UsableIndexes) of
[] ->
mango_sort_error(Db, Opts);
UsableIndexes ->
@@ -78,15 +76,8 @@ get_usable_indexes(Db, Selector, Opts) ->
end.
-mango_sort_error(Db, Opts) ->
- case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of
- {false, _} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index});
- {true, true} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned});
- {true, false} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_global})
- end.
+mango_sort_error(_Db, _Opts) ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index}).
recover(Db) ->
@@ -124,7 +115,6 @@ new(Db, Opts) ->
name = IdxName,
type = Type,
def = Def,
- partitioned = get_idx_partitioned(Opts),
opts = filter_opts(Opts)
}}.
@@ -136,11 +126,10 @@ validate_new(Idx, Db) ->
add(DDoc, Idx) ->
Mod = idx_mod(Idx),
- {ok, NewDDoc1} = Mod:add(DDoc, Idx),
- NewDDoc2 = set_ddoc_partitioned(NewDDoc1, Idx),
+ {ok, NewDDoc} = Mod:add(DDoc, Idx),
% Round trip through JSON for normalization
- Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)),
- {ok, NewDDoc2#doc{body = Body}}.
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
+ {ok, NewDDoc#doc{body = Body}}.
remove(DDoc, Idx) ->
@@ -192,8 +181,7 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc,
- partitioned = get_idx_partitioned(Db, Props)
+ ddoc = DDoc
}
end, Idxs).
@@ -230,10 +218,6 @@ def(#idx{def=Def}) ->
Def.
-partitioned(#idx{partitioned=Partitioned}) ->
- Partitioned.
-
-
opts(#idx{opts=Opts}) ->
Opts.
@@ -350,97 +334,6 @@ gen_name(Idx, Opts0) ->
mango_util:enc_hex(Sha).
-get_idx_partitioned(Opts) ->
- case proplists:get_value(partitioned, Opts) of
- B when is_boolean(B) ->
- B;
- db_default ->
- % Default to the partitioned setting on
- % the database.
- undefined
- end.
-
-
-set_ddoc_partitioned(DDoc, Idx) ->
- % We have to verify that the new index being added
- % to this design document either matches the current
- % ddoc's design options *or* this is a new design doc
- #doc{
- id = DDocId,
- revs = Revs,
- body = {BodyProps}
- } = DDoc,
- OldDOpts = couch_util:get_value(<<"options">>, BodyProps),
- OldOpt = case OldDOpts of
- {OldDOptProps} when is_list(OldDOptProps) ->
- couch_util:get_value(<<"partitioned">>, OldDOptProps);
- _ ->
- undefined
- end,
- % If new matches old we're done
- if Idx#idx.partitioned == OldOpt -> DDoc; true ->
- % If we're creating a ddoc then we can set the options
- case Revs == {0, []} of
- true when Idx#idx.partitioned /= undefined ->
- set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned);
- true when Idx#idx.partitioned == undefined ->
- DDoc;
- false ->
- ?MANGO_ERROR({partitioned_option_mismatch, DDocId})
- end
- end.
-
-
-set_ddoc_partitioned_option(DDoc, Partitioned) ->
- #doc{
- body = {BodyProps}
- } = DDoc,
- NewProps = case couch_util:get_value(<<"options">>, BodyProps) of
- {Existing} when is_list(Existing) ->
- Opt = {<<"partitioned">>, Partitioned},
- New = lists:keystore(<<"partitioned">>, 1, Existing, Opt),
- lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New});
- undefined ->
- New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}},
- lists:keystore(<<"options">>, 1, BodyProps, New)
- end,
- DDoc#doc{body = {NewProps}}.
-
-
-get_idx_partitioned(Db, DDocProps) ->
- Default = fabric_util:is_partitioned(Db),
- case couch_util:get_value(<<"options">>, DDocProps) of
- {DesignOpts} ->
- case couch_util:get_value(<<"partitioned">>, DesignOpts) of
- P when is_boolean(P) ->
- P;
- undefined ->
- Default
- end;
- undefined ->
- Default
- end.
-
-is_opts_partitioned(Opts) ->
- case couch_util:get_value(partition, Opts, <<>>) of
- <<>> ->
- false;
- Partition when is_binary(Partition) ->
- true
- end.
-
-
-filter_partition_indexes(Indexes, Opts) ->
- PFilt = case is_opts_partitioned(Opts) of
- false ->
- fun(#idx{partitioned = P}) -> not P end;
- true ->
- fun(#idx{partitioned = P}) -> P end
- end,
- Filt = fun(Idx) -> type(Idx) == <<"special">> orelse PFilt(Idx) end,
- lists:filter(Filt, Indexes).
-
-
filter_opts([]) ->
[];
filter_opts([{user_ctx, _} | Rest]) ->
@@ -453,8 +346,6 @@ filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{w, _} | Rest]) ->
filter_opts(Rest);
-filter_opts([{partitioned, _} | Rest]) ->
- filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
@@ -488,7 +379,6 @@ index(SelectorName, Selector) ->
<<"Selected">>,<<"json">>,
{[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
{SelectorName,{Selector}}]},
- false,
[{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
}.
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 9725950..712031b 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,6 +16,5 @@
name,
type,
def,
- partitioned,
opts
}).
diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl
index 1d4becf..71eaf11 100644
--- a/src/mango/src/mango_idx_text.erl
+++ b/src/mango/src/mango_idx_text.erl
@@ -100,7 +100,6 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {partitioned, Idx#idx.partitioned},
{def, {def_to_json(Idx#idx.def)}}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 3791149..2d784b6 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -104,7 +104,6 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {partitioned, Idx#idx.partitioned},
{def, {def_to_json(Idx#idx.def)}}
]}.
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 92c07f7..7bae9c9 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -34,7 +34,6 @@
validate_sort/1,
validate_fields/1,
validate_bulk_delete/1,
- validate_partitioned/1,
default_limit/0
]).
@@ -71,12 +70,6 @@ validate_idx_create({Props}) ->
{optional, true},
{default, 2},
{validator, fun is_pos_integer/1}
- ]},
- {<<"partitioned">>, [
- {tag, partitioned},
- {optional, true},
- {default, db_default},
- {validator, fun validate_partitioned/1}
]}
],
validate(Props, Opts).
@@ -124,12 +117,6 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
- {<<"partition">>, [
- {tag, partition},
- {optional, true},
- {default, <<>>},
- {validator, fun validate_partition/1}
- ]},
{<<"r">>, [
{tag, r},
{optional, true},
@@ -309,23 +296,6 @@ validate_fields(Value) ->
mango_fields:new(Value).
-validate_partitioned(true) ->
- {ok, true};
-validate_partitioned(false) ->
- {ok, false};
-validate_partitioned(db_default) ->
- {ok, db_default};
-validate_partitioned(Else) ->
- ?MANGO_ERROR({invalid_partitioned_value, Else}).
-
-
-validate_partition(<<>>) ->
- {ok, <<>>};
-validate_partition(Partition) ->
- couch_partition:validate_partition(Partition),
- {ok, Partition}.
-
-
validate_opts([], Props, Acc) ->
{Props, lists:reverse(Acc)};
validate_opts([{Name, Desc} | Rest], Props, Acc) ->
[couchdb] 13/18: Remove quorum stats
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a2fc004b69512c97fbcedb46f8bbd0db9b41bdaf
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:04:41 2020 +0200
Remove quorum stats
Removing quorum stats since they are not relevant with FDB.
---
src/mango/src/mango_cursor.erl | 5 +----
src/mango/src/mango_execution_stats.erl | 8 --------
src/mango/src/mango_execution_stats.hrl | 1 -
3 files changed, 1 insertion(+), 13 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index f16765b..db4e981 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -205,12 +205,9 @@ invalid_index_warning_int(_, _) ->
% returned, implying a lot of in-memory filtering
index_scan_warning(#execution_stats {
totalDocsExamined = Docs,
- totalQuorumDocsExamined = DocsQuorum,
resultsReturned = ResultCount
}) ->
- % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
- DocsScanned = Docs + DocsQuorum,
- Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount),
+ Ratio = calculate_index_scan_ratio(Docs, ResultCount),
Threshold = config:get_integer("mango", "index_scan_warning_threshold", 10),
case Threshold > 0 andalso Ratio > Threshold of
true ->
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 5878a31..fe9d27b 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -18,7 +18,6 @@
incr_keys_examined/1,
incr_docs_examined/1,
incr_docs_examined/2,
- incr_quorum_docs_examined/1,
incr_results_returned/1,
log_start/1,
log_end/1,
@@ -33,7 +32,6 @@ to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
- {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
]}.
@@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) ->
}.
-incr_quorum_docs_examined(Stats) ->
- Stats#execution_stats {
- totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
- }.
-
-
incr_results_returned(Stats) ->
couch_stats:increment_counter([mango, results_returned]),
Stats#execution_stats {
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
index ea5ed5e..783c1e7 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -13,7 +13,6 @@
-record(execution_stats, {
totalKeysExamined = 0,
totalDocsExamined = 0,
- totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
executionTimeMs
[couchdb] 16/18: Update mango test creds to same as elixir tests
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 370e490cba874d42982cb0c13d804012c35cc523
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 11:39:32 2020 +0200
Update mango test creds to same as elixir tests
---
Makefile | 2 +-
src/mango/test/README.md | 4 ++--
src/mango/test/mango.py | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index b3eb64c..9622c0e 100644
--- a/Makefile
+++ b/Makefile
@@ -349,7 +349,7 @@ mango-test: devclean all
@cd src/mango && \
python3 -m venv .venv && \
.venv/bin/python3 -m pip install -r requirements.txt
- @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=testuser:testpass '.venv/bin/python3 -m nose --with-xunit'
+ @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=adm:pass '.venv/bin/python3 -m nose --with-xunit'
################################################################################
# Developing
diff --git a/src/mango/test/README.md b/src/mango/test/README.md
index 509e32e..08693a2 100644
--- a/src/mango/test/README.md
+++ b/src/mango/test/README.md
@@ -23,7 +23,7 @@ Test configuration
The following environment variables can be used to configure the test fixtures:
* `COUCH_HOST` - root url (including port) of the CouchDB instance to run the tests against. Default is `"http://127.0.0.1:15984"`.
- * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"testuser"`.
- * `COUCH_PASSWORD` - CouchDB password. Default is `"testpass"`.
+ * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"adm"`.
+ * `COUCH_PASSWORD` - CouchDB password. Default is `"pass"`.
* `COUCH_AUTH_HEADER` - Optional Authorization header value. If specified, this is used instead of basic authentication with the username/password variables above.
* `MANGO_TEXT_INDEXES` - Set to `"1"` to run the tests only applicable to text indexes.
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 638de47..db0fab0 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -48,8 +48,8 @@ class Database(object):
dbname,
host="127.0.0.1",
port="15984",
- user="testuser",
- password="testpass",
+ user="adm",
+ password="pass",
):
root_url = get_from_environment("COUCH_HOST", "http://{}:{}".format(host, port))
auth_header = get_from_environment("COUCH_AUTH_HEADER", None)
[couchdb] 04/18: add fabric2 after_doc_write plugin
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bd192ea49c7ea6e1f46888c6b8f8fc2039009748
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Mar 19 17:05:54 2020 +0200
add fabric2 after_doc_write plugin
---
src/fabric/src/fabric2_db_plugin.erl | 5 +++++
src/fabric/src/fabric2_fdb.erl | 3 +++
2 files changed, 8 insertions(+)
diff --git a/src/fabric/src/fabric2_db_plugin.erl b/src/fabric/src/fabric2_db_plugin.erl
index fb83ed4..1d923dd 100644
--- a/src/fabric/src/fabric2_db_plugin.erl
+++ b/src/fabric/src/fabric2_db_plugin.erl
@@ -15,6 +15,7 @@
-export([
validate_dbname/3,
before_doc_update/3,
+ after_doc_write/6,
after_doc_read/2,
validate_docid/1,
check_is_admin/1,
@@ -49,6 +50,10 @@ before_doc_update(Db, Doc0, UpdateType) ->
end.
+after_doc_write(Db, Doc, NewWinner, OldWinner, NewRevId, Seq)->
+ with_pipe(after_doc_write, [Db, Doc, NewWinner, OldWinner, NewRevId, Seq]).
+
+
after_doc_read(Db, Doc0) ->
Fun = fabric2_db:get_after_doc_read_fun(Db),
case with_pipe(after_doc_read, [Doc0, Db]) of
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 22ccc99..912d4df 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -832,6 +832,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ fabric2_db_plugin:after_doc_write(Db, Doc, NewWinner, OldWinner,
+ NewRevId, WinnerVS),
+
% Update database size
AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
RemSize = sum_rem_rev_sizes(ToRemove),
[couchdb] 06/18: All couch_view queries to run across transactions
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5000260dc9ea71ab33abe2c766dc2bfab75167b8
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 25 15:10:00 2020 +0200
All couch_view queries to run across transactions
---
src/couch_views/src/couch_views_reader.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 76dbed1..ce7f163 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -184,7 +184,8 @@ mrargs_to_fdb_options(Args) ->
[
{dir, Direction},
{limit, Limit + Skip},
- {streaming_mode, want_all}
+ {streaming_mode, want_all},
+ {restart_tx, true}
] ++ StartKeyOpts ++ EndKeyOpts.
[couchdb] 18/18: remove defer and load_ddocs from mango_utils
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 95477cc0a791a139581ed2bbfe143befc211dca1
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 15:26:03 2020 +0200
remove defer and load_ddocs from mango_utils
Clean up unused mango_utils functions.
---
src/mango/src/mango_idx.erl | 16 ---------------
src/mango/src/mango_util.erl | 47 --------------------------------------------
2 files changed, 63 deletions(-)
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 99a3588..9907cc7 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -19,7 +19,6 @@
-export([
list/1,
- recover/1,
new/2,
validate_new/2,
@@ -105,21 +104,6 @@ mango_sort_error(_Db, _Opts) ->
?MANGO_ERROR({no_usable_index, missing_sort_index}).
-recover(Db) ->
- {ok, DDocs0} = mango_util:open_ddocs(Db),
- Pred = fun({Props}) ->
- case proplists:get_value(<<"language">>, Props) of
- <<"query">> -> true;
- _ -> false
- end
- end,
- DDocs = lists:filter(Pred, DDocs0),
- Special = special(Db),
- {ok, Special ++ lists:flatmap(fun(Doc) ->
- from_ddoc(Db, Doc)
- end, DDocs)}.
-
-
get_sort_fields(Opts) ->
case lists:keyfind(sort, 1, Opts) of
{sort, Sort} ->
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 18a6439..d649f95 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -15,13 +15,9 @@
-export([
open_doc/2,
- open_ddocs/1,
load_ddoc/2,
load_ddoc/3,
- defer/3,
- do_defer/3,
-
assert_ejson/1,
to_lower/1,
@@ -88,15 +84,6 @@ open_doc(Db, DocId, Options) ->
fabric2_db:open_doc(Db, DocId, Options).
-open_ddocs(Db) ->
- case mango_util:defer(fabric, design_docs, [Db]) of
- {ok, Docs} ->
- {ok, Docs};
- _ ->
- ?MANGO_ERROR(error_loading_ddocs)
- end.
-
-
load_ddoc(Db, DDocId) ->
load_ddoc(Db, DDocId, [deleted, ejson_body]).
@@ -112,40 +99,6 @@ load_ddoc(Db, DDocId, DbOpts) ->
end.
-defer(Mod, Fun, Args) ->
- {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]),
- receive
- {'DOWN', Ref, process, Pid, {mango_defer_ok, Value}} ->
- Value;
- {'DOWN', Ref, process, Pid, {mango_defer_throw, Value}} ->
- erlang:throw(Value);
- {'DOWN', Ref, process, Pid, {mango_defer_error, Value}} ->
- erlang:error(Value);
- {'DOWN', Ref, process, Pid, {mango_defer_exit, Value}} ->
- erlang:exit(Value)
- end.
-
-
-do_defer(Mod, Fun, Args) ->
- try erlang:apply(Mod, Fun, Args) of
- Resp ->
- erlang:exit({mango_defer_ok, Resp})
- catch
- throw:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{throw, Error}, Stack]),
- erlang:exit({mango_defer_throw, Error});
- error:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{error, Error}, Stack]),
- erlang:exit({mango_defer_error, Error});
- exit:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{exit, Error}, Stack]),
- erlang:exit({mango_defer_exit, Error})
- end.
-
-
assert_ejson({Props}) ->
assert_ejson_obj(Props);
assert_ejson(Vals) when is_list(Vals) ->
[couchdb] 17/18: update mango tests to work with Mango on FDB
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 36ed475e41c0b58aad4fb310660cc03704fac0aa
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 11:39:58 2020 +0200
update mango tests to work with Mango on FDB
---
Makefile | 3 +-
src/mango/test/01-index-crud-test.py | 15 ++++++++
src/mango/test/02-basic-find-test.py | 15 --------
src/mango/test/05-index-selection-test.py | 7 +++-
src/mango/test/12-use-correct-index-test.py | 38 +++++++++++--------
src/mango/test/13-stable-update-test.py | 51 --------------------------
src/mango/test/13-users-db-find-test.py | 7 +++-
src/mango/test/15-execution-stats-test.py | 7 +---
src/mango/test/17-multi-type-value-test.py | 4 +-
src/mango/test/19-find-conflicts.py | 7 ++--
src/mango/test/20-no-timeout-test.py | 32 ----------------
src/mango/test/22-build-wait-selected-index.py | 50 +++++++++++++++++++++++++
src/mango/test/mango.py | 20 +++++++---
src/mango/test/user_docs.py | 7 ++--
14 files changed, 128 insertions(+), 135 deletions(-)
diff --git a/Makefile b/Makefile
index 9622c0e..1326d40 100644
--- a/Makefile
+++ b/Makefile
@@ -160,9 +160,10 @@ endif
.PHONY: check-fdb
check-fdb:
- make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric
+ make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric,mango
make elixir tests=test/elixir/test/basics_test.exs,test/elixir/test/replication_test.exs,test/elixir/test/map_test.exs,test/elixir/test/all_docs_test.exs,test/elixir/test/bulk_docs_test.exs
make exunit tests=src/couch_rate/test/exunit/
+ make mango-test
.PHONY: eunit
# target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options
diff --git a/src/mango/test/01-index-crud-test.py b/src/mango/test/01-index-crud-test.py
index b602399..13ae300 100644
--- a/src/mango/test/01-index-crud-test.py
+++ b/src/mango/test/01-index-crud-test.py
@@ -113,6 +113,21 @@ class IndexCrudTests(mango.DbPerClass):
return
raise AssertionError("index not created")
+ def test_ignore_design_docs(self):
+ fields = ["baz", "foo"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+ self.db.save_doc({
+ "_id": "_design/ignore",
+ "views": {
+ "view1": {
+ "map": "function (doc) { emit(doc._id, 1)}"
+ }
+ }
+ })
+ indexes = self.db.list_indexes()
+ self.assertEqual(len(indexes), 2)
+
def test_read_idx_doc(self):
self.db.create_index(["foo", "bar"], name="idx_01")
self.db.create_index(["hello", "bar"])
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index afdba03..2a03a3a 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -100,16 +100,6 @@ class BasicFindTests(mango.UserDocsTests):
else:
raise AssertionError("bad find")
- def test_bad_r(self):
- bad_rs = ([None, True, False, 1.2, "no limit!", {"foo": "bar"}, [2]],)
- for br in bad_rs:
- try:
- self.db.find({"int": {"$gt": 2}}, r=br)
- except Exception as e:
- assert e.response.status_code == 400
- else:
- raise AssertionError("bad find")
-
def test_bad_conflicts(self):
bad_conflicts = ([None, 1.2, "no limit!", {"foo": "bar"}, [2]],)
for bc in bad_conflicts:
@@ -262,11 +252,6 @@ class BasicFindTests(mango.UserDocsTests):
assert sorted(d.keys()) == ["location", "user_id"]
assert sorted(d["location"].keys()) == ["address"]
- def test_r(self):
- for r in [1, 2, 3]:
- docs = self.db.find({"age": {"$gt": 0}}, r=r)
- assert len(docs) == 15
-
def test_empty(self):
docs = self.db.find({})
# 15 users
diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py
index cb4d329..bae3d58 100644
--- a/src/mango/test/05-index-selection-test.py
+++ b/src/mango/test/05-index-selection-test.py
@@ -14,6 +14,8 @@ import mango
import user_docs
import unittest
+import requests
+
class IndexSelectionTests:
def test_basic(self):
@@ -201,8 +203,11 @@ class IndexSelectionTests:
}
},
}
- with self.assertRaises(KeyError):
+ try:
self.db.save_doc(design_doc)
+ assert False, "Should not get here."
+ except requests.exceptions.HTTPError as e:
+ self.assertEqual(e.response.json()['error'], 'invalid_design_doc')
def test_explain_sort_reverse(self):
selector = {"manager": {"$gt": None}}
diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py
index c21ad6c..a7f07b5 100644
--- a/src/mango/test/12-use-correct-index-test.py
+++ b/src/mango/test/12-use-correct-index-test.py
@@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs(copy.deepcopy(DOCS))
def test_choose_index_with_one_field_in_index(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie"}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
def test_choose_index_with_two(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_alphabetically(self):
- self.db.create_index(["name"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/aaa")
def test_choose_index_most_accurate(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_most_accurate_in_memory_selector(self):
- self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
@@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
def test_chooses_idxA(self):
DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}]
self.db.save_docs(copy.deepcopy(DOCS2))
- self.db.create_index(["a", "b", "c"])
- self.db.create_index(["a", "d", "e"])
+ self.db.create_index(["a", "b", "c"], wait_for_built_index=False)
+ self.db.create_index(["a", "d", "e"], wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find(
{"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True
)
diff --git a/src/mango/test/13-stable-update-test.py b/src/mango/test/13-stable-update-test.py
deleted file mode 100644
index 303f3fa..0000000
--- a/src/mango/test/13-stable-update-test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import copy
-import mango
-
-DOCS1 = [
- {
- "_id": "54af50626de419f5109c962f",
- "user_id": 0,
- "age": 10,
- "name": "Jimi",
- "location": "UK",
- "number": 4,
- },
- {
- "_id": "54af50622071121b25402dc3",
- "user_id": 1,
- "age": 12,
- "name": "Eddie",
- "location": "ZAR",
- "number": 2,
- },
-]
-
-
-class SupportStableAndUpdate(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- # Hack to prevent auto-indexer from foiling update=False test
- # https://github.com/apache/couchdb/issues/2313
- self.db.save_doc(
- {"_id": "_design/foo", "language": "query", "autoupdate": False}
- )
- self.db.create_index(["name"], ddoc="foo")
- self.db.save_docs(copy.deepcopy(DOCS1))
-
- def test_update_updates_view_when_specified(self):
- docs = self.db.find({"name": "Eddie"}, update=False)
- assert len(docs) == 0
- docs = self.db.find({"name": "Eddie"}, update=True)
- assert len(docs) == 1
diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py
index 73d15ea..9f9b53a 100644
--- a/src/mango/test/13-users-db-find-test.py
+++ b/src/mango/test/13-users-db-find-test.py
@@ -12,10 +12,15 @@
# the License.
-import mango, requests
+import mango, requests, unittest
+# Re-enable once the _users db is implemented
class UsersDbFindTests(mango.UsersDbTests):
+ @classmethod
+ def setUpClass(klass):
+ raise unittest.SkipTest("Re-enable once the _users db is implemented")
+
def test_simple_find(self):
docs = self.db.find({"name": {"$eq": "demo02"}})
assert len(docs) == 1
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 537a19a..6ccc04b 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -35,12 +34,11 @@ class ExecutionStatsTests(mango.UserDocsTests):
def test_quorum_json_index(self):
resp = self.db.find(
- {"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True
+ {"age": {"$lt": 35}}, return_raw=True, executionStats=True
)
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -70,7 +68,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
self.assertEqual(len(resp["docs"]), 1)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
index 21e7afd..5a8fced 100644
--- a/src/mango/test/17-multi-type-value-test.py
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -53,9 +53,9 @@ class MultiValueFieldTests:
class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["name"], wait_for_built_index=False)
+ self.db.create_index(["age", "name"], wait_for_built_index=True)
self.db.save_docs(copy.deepcopy(DOCS))
- self.db.create_index(["name"])
- self.db.create_index(["age", "name"])
# @unittest.skipUnless(mango.has_text_service(), "requires text service")
diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py
index bf865d6..3bf3c06 100644
--- a/src/mango/test/19-find-conflicts.py
+++ b/src/mango/test/19-find-conflicts.py
@@ -12,11 +12,12 @@
import mango
import copy
+import unittest
-DOC = [{"_id": "doc", "a": 2}]
+DOC = [{"_id": "doc", "a": 2}, {"_id": "doc1", "b": 2}]
CONFLICT = [{"_id": "doc", "_rev": "1-23202479633c2b380f79507a776743d5", "a": 1}]
-
+CONFLICT2 = [{"_id": "doc1", "_rev": "1-23202479633c2b380f79507a776743d5", "b": 1}]
class ChooseCorrectIndexForDocs(mango.DbPerClass):
def setUp(self):
@@ -25,7 +26,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs_with_conflicts(copy.deepcopy(CONFLICT))
def test_retrieve_conflicts(self):
- self.db.create_index(["_conflicts"])
+ self.db.create_index(["_conflicts"], wait_for_built_index=False)
result = self.db.find({"_conflicts": {"$exists": True}}, conflicts=True)
self.assertEqual(
result[0]["_conflicts"][0], "1-23202479633c2b380f79507a776743d5"
diff --git a/src/mango/test/20-no-timeout-test.py b/src/mango/test/20-no-timeout-test.py
deleted file mode 100644
index cffdfc3..0000000
--- a/src/mango/test/20-no-timeout-test.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import mango
-import copy
-import unittest
-
-
-class LongRunningMangoTest(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- docs = []
- for i in range(100000):
- docs.append({"_id": str(i), "another": "field"})
- if i % 20000 == 0:
- self.db.save_docs(docs)
- docs = []
-
- # This test should run to completion and not timeout
- def test_query_does_not_time_out(self):
- selector = {"_id": {"$gt": 0}, "another": "wrong"}
- docs = self.db.find(selector)
- self.assertEqual(len(docs), 0)
diff --git a/src/mango/test/22-build-wait-selected-index.py b/src/mango/test/22-build-wait-selected-index.py
new file mode 100644
index 0000000..fd856f4
--- /dev/null
+++ b/src/mango/test/22-build-wait-selected-index.py
@@ -0,0 +1,50 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import copy
+import unittest
+
+
+class BuildAndWaitOnSelectedIndex(mango.DbPerClass):
+ def setUp(self):
+ self.db.recreate()
+ docs = []
+ for i in range(1000):
+ docs.append({"_id": str(i), "val": i})
+ if len(docs) == 250:
+ self.db.save_docs(docs)
+ docs = []
+
+ def test_wait_for_query(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+
+ explain = self.db.find({'val': {"$gt": 990}}, use_index="my-ddoc", explain=True)
+ self.assertEqual(explain["index"]["ddoc"], "_design/my-ddoc")
+
+ docs = self.db.find({'val': {"$gte": 990}}, limit=10)
+
+ self.assertEqual(len(docs), 10)
+
+ def test_dont_wait(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+
+ explain = self.db.find({'val': {"$gt": 990}}, explain=True)
+ self.assertEqual(explain["index"]["name"], "_all_docs")
+
+ docs = self.db.find({'val': {"$gte": 990}})
+ self.assertEqual(len(docs), 10)
+
+ def test_update_false(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+ docs = self.db.find({'val': {"$gte": 990}}, update=False, use_index="my-ddoc")
+ self.assertEqual(docs, [])
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index db0fab0..e013e0e 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -139,8 +139,9 @@ class Database(object):
ddoc=None,
partial_filter_selector=None,
selector=None,
+ wait_for_built_index=True,
):
- body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
+ body = {"index": {"fields": fields}, "type": idx_type}
if name is not None:
body["name"] = name
if ddoc is not None:
@@ -156,13 +157,22 @@ class Database(object):
assert r.json()["name"] is not None
created = r.json()["result"] == "created"
- if created:
- # wait until the database reports the index as available
- while len(self.get_index(r.json()["id"], r.json()["name"])) < 1:
- delay(t=0.1)
+ if created and wait_for_built_index:
+ # wait until the database reports the index as available and build
+ while True:
+ idx = self.get_index(r.json()["id"], r.json()["name"])[0]
+ if idx["build_status"] == "ready":
+ break
+ delay(t=0.2)
return created
+ def wait_for_built_indexes(self):
+ while True:
+ if all(idx["build_status"] == "ready" for idx in self.list_indexes()):
+ break
+ delay(t=0.2)
+
def create_text_index(
self,
analyzer=None,
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 8f0ed2e..d69e6d6 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -61,12 +61,11 @@ def setup_users(db, **kwargs):
def setup(db, index_type="view", **kwargs):
db.recreate()
- db.save_docs(copy.deepcopy(DOCS))
if index_type == "view":
add_view_indexes(db, kwargs)
elif index_type == "text":
add_text_indexes(db, kwargs)
-
+ db.save_docs(copy.deepcopy(DOCS))
def add_view_indexes(db, kwargs):
indexes = [
@@ -90,7 +89,9 @@ def add_view_indexes(db, kwargs):
(["ordered"], "ordered"),
]
for (idx, name) in indexes:
- assert db.create_index(idx, name=name, ddoc=name) is True
+ assert db.create_index(idx, name=name, ddoc=name,
+ wait_for_built_index=False) is True
+ db.wait_for_built_indexes()
def add_text_indexes(db, kwargs):
[couchdb] 12/18: Remove view_cb predicate push down
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 825aafc198681be673fa9de67129d0da116aa6fd
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:11:03 2020 +0200
Remove view_cb predicate push down
Removes the view callback that was performed on the nodes before
sending the results back to the co-ordinator.
---
src/mango/src/mango_cursor_view.erl | 95 +------------------------------------
1 file changed, 1 insertion(+), 94 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 5187c10..b88f6ea 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,7 +19,6 @@
]).
-export([
- view_cb/2,
handle_message/2,
handle_all_docs_message/2,
composite_indexes/2,
@@ -114,8 +113,7 @@ base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
reduce = false,
start_key = StartKey,
end_key = EndKey,
- include_docs = true,
- extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
+ include_docs = true
}.
@@ -226,66 +224,6 @@ choose_best_index(_DbName, IndexRanges) ->
{SelectedIndex, SelectedIndexRanges}.
-view_cb({meta, Meta}, Acc) ->
- % Map function starting
- put(mango_docs_examined, 0),
- set_mango_msg_timestamp(),
- ok = rexi:stream2({meta, Meta}),
- {ok, Acc};
-view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
- ViewRow = #view_row{
- id = couch_util:get_value(id, Row),
- key = couch_util:get_value(key, Row),
- doc = couch_util:get_value(doc, Row)
- },
- case ViewRow#view_row.doc of
- null ->
- maybe_send_mango_ping();
- undefined ->
- % include_docs=false. Use quorum fetch at coordinator
- ok = rexi:stream2(ViewRow),
- set_mango_msg_timestamp();
- Doc ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
- Selector = couch_util:get_value(selector, Options),
- couch_stats:increment_counter([mango, docs_examined]),
- case mango_selector:match(Selector, Doc) of
- true ->
- ok = rexi:stream2(ViewRow),
- set_mango_msg_timestamp();
- false ->
- maybe_send_mango_ping()
- end
- end,
- {ok, Acc};
-view_cb(complete, Acc) ->
- % Send shard-level execution stats
- ok = rexi:stream2({execution_stats, {docs_examined, get(mango_docs_examined)}}),
- % Finish view output
- ok = rexi:stream_last(complete),
- {ok, Acc};
-view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
-
-
-maybe_send_mango_ping() ->
- Current = os:timestamp(),
- LastPing = get(mango_last_msg_timestamp),
- % Fabric will timeout if it has not heard a response from a worker node
- % after 5 seconds. Send a ping every 4 seconds so the timeout doesn't happen.
- case timer:now_diff(Current, LastPing) > ?HEARTBEAT_INTERVAL_IN_USEC of
- false ->
- ok;
- true ->
- rexi:ping(),
- set_mango_msg_timestamp()
- end.
-
-
-set_mango_msg_timestamp() ->
- put(mango_last_msg_timestamp, os:timestamp()).
-
-
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
@@ -456,34 +394,3 @@ update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
};
update_bookmark_keys(Cursor, _Props) ->
Cursor.
-
-
-%%%%%%%% module tests below %%%%%%%%
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-
-does_not_refetch_doc_with_value_test() ->
- Cursor = #cursor {
- db = <<"db">>,
- opts = [],
- execution_stats = #execution_stats{},
- selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]})
- },
- RowProps = [
- {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {doc,{
- [
- {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>},
- {<<"user_id">>,11}
- ]
- }}
- ],
- {Match, _, _} = doc_member(Cursor, RowProps),
- ?assertEqual(Match, ok).
-
-
--endif.
[couchdb] 01/18: fix all_docs call to return row
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a553bc2f87a60bdc0eb98254284355e1ca0dd1e2
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 18 09:39:50 2020 +0200
fix all_docs call to return row
---
src/chttpd/src/chttpd_db.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index dea992c..174d464 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -985,7 +985,8 @@ apply_args_to_keylist(Args, Keys0) ->
view_cb({row, Row}, {iter, Db, Args, VAcc}) ->
NewRow = case lists:keymember(doc, 1, Row) of
true ->
- chttpd_stats:incr_reads();
+ chttpd_stats:incr_reads(),
+ Row;
false when Args#mrargs.include_docs ->
{id, DocId} = lists:keyfind(id, 1, Row),
chttpd_stats:incr_reads(),
[couchdb] 11/18: remove unneeded r/w parameter
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 617037226f392e4b9c74ace1f440c955e80eea0a
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 15:35:22 2020 +0200
remove unneeded r/w parameter
---
src/mango/src/mango_crud.erl | 17 +++--------------
src/mango/src/mango_cursor_view.erl | 12 ------------
src/mango/src/mango_httpd.erl | 28 +++-------------------------
src/mango/src/mango_idx.erl | 2 --
src/mango/src/mango_opts.erl | 12 ------------
src/mango/test/mango.py | 2 --
6 files changed, 6 insertions(+), 67 deletions(-)
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 41a4d14..42717ff 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -35,8 +35,7 @@ insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, Docs, Opts0) when is_list(Docs) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(w, Opts1),
- case fabric:update_docs(Db, Docs, Opts2) of
+ case fabric:update_docs(Db, Docs, Opts1) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -48,8 +47,7 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
find(Db, Selector, Callback, UserAcc, Opts0) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -101,8 +99,7 @@ delete(Db, Selector, Options) ->
explain(Db, Selector, Opts0) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
mango_cursor:explain(Cursor).
@@ -115,14 +112,6 @@ maybe_add_user_ctx(Db, Opts) ->
end.
-maybe_int_to_str(_Key, []) ->
- [];
-maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) ->
- [{Key, integer_to_list(Val)} | maybe_int_to_str(Key, Rest)];
-maybe_int_to_str(Key, [KV | Rest]) ->
- [KV | maybe_int_to_str(Key, Rest)].
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index bced842..5187c10 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -353,18 +353,6 @@ ddocid(Idx) ->
apply_opts([], Args) ->
Args;
-apply_opts([{r, RStr} | Rest], Args) ->
- IncludeDocs = case list_to_integer(RStr) of
- 1 ->
- true;
- R when R > 1 ->
- % We don't load the doc in the view query because
- % we have to do a quorum read in the coordinator
- % so there's no point.
- false
- end,
- NewArgs = Args#mrargs{include_docs = IncludeDocs},
- apply_opts(Rest, NewArgs);
apply_opts([{conflicts, true} | Rest], Args) ->
NewArgs = Args#mrargs{conflicts = true},
apply_opts(Rest, NewArgs);
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 946d7e4..1054c74 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -94,8 +94,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, DDoc} ->
<<"exists">>;
{ok, NewDDoc} ->
- CreateOpts = get_idx_w_opts(Opts),
- case mango_crud:insert(Db, NewDDoc, CreateOpts) of
+ case mango_crud:insert(Db, NewDDoc, Opts) of
{ok, [{RespProps}]} ->
case lists:keyfind(error, 1, RespProps) of
{error, Reason} ->
@@ -121,12 +120,11 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
{ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
Idxs = mango_idx:list(Db),
DDocs = get_bulk_delete_ddocs(Opts),
- DelOpts = get_idx_w_opts(Opts),
{Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
DDocId = convert_to_design_id(DDocId0),
Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
Id = {<<"id">>, DDocId},
- case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ case mango_idx:delete(Filt, Db, Idxs, Opts) of
{ok, true} ->
{[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
{error, Error} ->
@@ -148,14 +146,13 @@ handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
Idxs = mango_idx:list(Db),
DDocId = convert_to_design_id(DDocId0),
- DelOpts = get_idx_del_opts(Req),
Filt = fun(Idx) ->
IsDDoc = mango_idx:ddoc(Idx) == DDocId,
IsType = mango_idx:type(Idx) == Type,
IsName = mango_idx:name(Idx) == Name,
IsDDoc andalso IsType andalso IsName
end,
- case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ case mango_idx:delete(Filt, Db, Idxs, []) of
{ok, true} ->
chttpd:send_json(Req, {[{ok, true}]});
{error, not_found} ->
@@ -203,15 +200,6 @@ set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
NewDb.
-get_idx_w_opts(Opts) ->
- case lists:keyfind(w, 1, Opts) of
- {w, N} when is_integer(N), N > 0 ->
- [{w, integer_to_list(N)}];
- _ ->
- [{w, "2"}]
- end.
-
-
get_bulk_delete_ddocs(Opts) ->
case lists:keyfind(docids, 1, Opts) of
{docids, DDocs} when is_list(DDocs) ->
@@ -221,16 +209,6 @@ get_bulk_delete_ddocs(Opts) ->
end.
-get_idx_del_opts(Req) ->
- try
- WStr = chttpd:qs_value(Req, "w", "2"),
- _ = list_to_integer(WStr),
- [{w, WStr}]
- catch _:_ ->
- [{w, "2"}]
- end.
-
-
convert_to_design_id(DDocId) ->
case DDocId of
<<"_design/", _/binary>> -> DDocId;
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 0f79bdb..a26a685 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -344,8 +344,6 @@ filter_opts([{name, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
-filter_opts([{w, _} | Rest]) ->
- filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 7bae9c9..e357676 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -64,12 +64,6 @@ validate_idx_create({Props}) ->
{optional, true},
{default, auto_name},
{validator, fun validate_idx_name/1}
- ]},
- {<<"w">>, [
- {tag, w},
- {optional, true},
- {default, 2},
- {validator, fun is_pos_integer/1}
]}
],
validate(Props, Opts).
@@ -117,12 +111,6 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
- {<<"r">>, [
- {tag, r},
- {optional, true},
- {default, 1},
- {validator, fun mango_opts:is_pos_integer/1}
- ]},
{<<"conflicts">>, [
{tag, conflicts},
{optional, true},
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 03cb85f..638de47 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -244,7 +244,6 @@ class Database(object):
skip=0,
sort=None,
fields=None,
- r=1,
conflicts=False,
use_index=None,
explain=False,
@@ -258,7 +257,6 @@ class Database(object):
"use_index": use_index,
"limit": limit,
"skip": skip,
- "r": r,
"conflicts": conflicts,
}
if sort is not None:
[couchdb] 03/18: add include_docs option to fold_docs
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ec50197a79b445249f2bc41793d001ff78c3ac9e
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 10 18:30:44 2020 +0200
add include_docs option to fold_docs
---
src/fabric/src/fabric2_db.erl | 27 +++++++++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index ca9f037..fb6ae51 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -832,11 +832,23 @@ fold_docs(Db, UserFun, UserAcc0, Options) ->
UserAcc2 = fabric2_fdb:fold_range(TxDb, Prefix, fun({K, V}, Acc) ->
{DocId} = erlfdb_tuple:unpack(K, Prefix),
RevId = erlfdb_tuple:unpack(V),
- maybe_stop(UserFun({row, [
+ Row0 = [
{id, DocId},
{key, DocId},
{value, {[{rev, couch_doc:rev_to_str(RevId)}]}}
- ]}, Acc))
+ ],
+
+ DocOpts = couch_util:get_value(doc_opts, Options, []),
+ OpenOpts = [deleted | DocOpts],
+
+ Row1 = case lists:keyfind(include_docs, 1, Options) of
+ {include_docs, true} ->
+ Row0 ++ open_json_doc(Db, DocId, OpenOpts, DocOpts);
+ _ ->
+ Row0
+ end,
+
+ maybe_stop(UserFun({row, Row1}, Acc))
end, UserAcc1, Options),
{ok, maybe_stop(UserFun(complete, UserAcc2))}
@@ -1878,3 +1890,14 @@ stem_revisions(#{} = Db, #doc{} = Doc) ->
true -> Doc#doc{revs = {RevPos, lists:sublist(Revs, RevsLimit)}};
false -> Doc
end.
+
+
+open_json_doc(Db, DocId, OpenOpts, DocOpts) ->
+ case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ [];
+ {ok, #doc{deleted = true}} ->
+ [{doc, null}];
+ {ok, #doc{} = Doc} ->
+ [{doc, couch_doc:to_json_obj(Doc, DocOpts)}]
+ end.
[couchdb] 09/18: remove mango native proc
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit daecc5dd12ba8dd7f6d0fd8ead990f05cef45b91
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 10 18:20:27 2020 +0200
remove mango native proc
---
src/couch/src/couch_proc_manager.erl | 1 -
src/couch_js/src/couch_js_proc_manager.erl | 1 -
src/mango/src/mango_native_proc.erl | 373 -----------------------------
3 files changed, 375 deletions(-)
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 0daef3e..376e12e 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -109,7 +109,6 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl
index 0964696..45f1736 100644
--- a/src/couch_js/src/couch_js_proc_manager.erl
+++ b/src/couch_js/src/couch_js_proc_manager.erl
@@ -108,7 +108,6 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
deleted file mode 100644
index cbf3622..0000000
--- a/src/mango/src/mango_native_proc.erl
+++ /dev/null
@@ -1,373 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(mango_native_proc).
--behavior(gen_server).
-
-
--include("mango_idx.hrl").
-
-
--export([
- start_link/0,
- set_timeout/2,
- prompt/2
-]).
-
--export([
- init/1,
- terminate/2,
- handle_call/3,
- handle_cast/2,
- handle_info/2,
- code_change/3
-]).
-
-
--record(st, {
- indexes = [],
- timeout = 5000
-}).
-
-
--record(tacc, {
- index_array_lengths = true,
- fields = all_fields,
- path = []
-}).
-
-
-start_link() ->
- gen_server:start_link(?MODULE, [], []).
-
-
-set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
- gen_server:call(Pid, {set_timeout, TimeOut}).
-
-
-prompt(Pid, Data) ->
- gen_server:call(Pid, {prompt, Data}).
-
-
-init(_) ->
- {ok, #st{}}.
-
-
-terminate(_Reason, _St) ->
- ok.
-
-
-handle_call({set_timeout, TimeOut}, _From, St) ->
- {reply, ok, St#st{timeout=TimeOut}};
-
-handle_call({prompt, [<<"reset">>]}, _From, St) ->
- {reply, true, St#st{indexes=[]}};
-
-handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) ->
- {reply, true, St#st{indexes=[]}};
-
-handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
- Indexes = case validate_index_info(IndexInfo) of
- true ->
- St#st.indexes ++ [IndexInfo];
- false ->
- couch_log:error("No Valid Indexes For: ~p", [IndexInfo]),
- St#st.indexes
- end,
- NewSt = St#st{indexes = Indexes},
- {reply, true, NewSt};
-
-handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) ->
- {reply, map_doc(St, mango_json:to_binary(Doc)), St};
-
-handle_call({prompt, [<<"reduce">>, RedSrcs, _]}, _From, St) ->
- {reply, [true, [null || _ <- RedSrcs]], St};
-
-handle_call({prompt, [<<"rereduce">>, RedSrcs, _]}, _From, St) ->
- {reply, [true, [null || _ <- RedSrcs]], St};
-
-handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
- Vals = case index_doc(St, mango_json:to_binary(Doc)) of
- [] ->
- [[]];
- Else ->
- Else
- end,
- {reply, Vals, St};
-
-
-handle_call(Msg, _From, St) ->
- {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
-
-
-handle_cast(garbage_collect, St) ->
- erlang:garbage_collect(),
- {noreply, St};
-
-handle_cast(stop, St) ->
- {stop, normal, St};
-
-handle_cast(Msg, St) ->
- {stop, {invalid_cast, Msg}, St}.
-
-
-handle_info(Msg, St) ->
- {stop, {invalid_info, Msg}, St}.
-
-
-code_change(_OldVsn, St, _Extra) ->
- {ok, St}.
-
-
-map_doc(#st{indexes=Indexes}, Doc) ->
- lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes).
-
-
-index_doc(#st{indexes=Indexes}, Doc) ->
- lists:map(fun(Idx) -> get_text_entries(Idx, Doc) end, Indexes).
-
-
-get_index_entries({IdxProps}, Doc) ->
- {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
- Selector = get_index_partial_filter_selector(IdxProps),
- case should_index(Selector, Doc) of
- false ->
- [];
- true ->
- Values = get_index_values(Fields, Doc),
- case lists:member(not_found, Values) of
- true -> [];
- false -> [[Values, null]]
- end
- end.
-
-
-get_index_values(Fields, Doc) ->
- lists:map(fun({Field, _Dir}) ->
- case mango_doc:get_field(Doc, Field) of
- not_found -> not_found;
- bad_path -> not_found;
- Value -> Value
- end
- end, Fields).
-
-
-get_text_entries({IdxProps}, Doc) ->
- Selector = get_index_partial_filter_selector(IdxProps),
- case should_index(Selector, Doc) of
- true ->
- get_text_entries0(IdxProps, Doc);
- false ->
- []
- end.
-
-
-get_index_partial_filter_selector(IdxProps) ->
- case couch_util:get_value(<<"partial_filter_selector">>, IdxProps, {[]}) of
- {[]} ->
- % this is to support legacy text indexes that had the partial_filter_selector
- % set as selector
- couch_util:get_value(<<"selector">>, IdxProps, {[]});
- Else ->
- Else
- end.
-
-
-get_text_entries0(IdxProps, Doc) ->
- DefaultEnabled = get_default_enabled(IdxProps),
- IndexArrayLengths = get_index_array_lengths(IdxProps),
- FieldsList = get_text_field_list(IdxProps),
- TAcc = #tacc{
- index_array_lengths = IndexArrayLengths,
- fields = FieldsList
- },
- Fields0 = get_text_field_values(Doc, TAcc),
- Fields = if not DefaultEnabled -> Fields0; true ->
- add_default_text_field(Fields0)
- end,
- FieldNames = get_field_names(Fields),
- Converted = convert_text_fields(Fields),
- FieldNames ++ Converted.
-
-
-get_text_field_values({Props}, TAcc) when is_list(Props) ->
- get_text_field_values_obj(Props, TAcc, []);
-
-get_text_field_values(Values, TAcc) when is_list(Values) ->
- IndexArrayLengths = TAcc#tacc.index_array_lengths,
- NewPath = ["[]" | TAcc#tacc.path],
- NewTAcc = TAcc#tacc{path = NewPath},
- case IndexArrayLengths of
- true ->
- % We bypass make_text_field and directly call make_text_field_name
- % because the length field name is not part of the path.
- LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>),
- LengthField = [{LengthFieldName, <<"length">>, length(Values)}],
- get_text_field_values_arr(Values, NewTAcc, LengthField);
- _ ->
- get_text_field_values_arr(Values, NewTAcc, [])
- end;
-
-get_text_field_values(Bin, TAcc) when is_binary(Bin) ->
- make_text_field(TAcc, <<"string">>, Bin);
-
-get_text_field_values(Num, TAcc) when is_number(Num) ->
- make_text_field(TAcc, <<"number">>, Num);
-
-get_text_field_values(Bool, TAcc) when is_boolean(Bool) ->
- make_text_field(TAcc, <<"boolean">>, Bool);
-
-get_text_field_values(null, TAcc) ->
- make_text_field(TAcc, <<"null">>, true).
-
-
-get_text_field_values_obj([], _, FAcc) ->
- FAcc;
-get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) ->
- NewPath = [Key | TAcc#tacc.path],
- NewTAcc = TAcc#tacc{path = NewPath},
- Fields = get_text_field_values(Val, NewTAcc),
- get_text_field_values_obj(Rest, TAcc, Fields ++ FAcc).
-
-
-get_text_field_values_arr([], _, FAcc) ->
- FAcc;
-get_text_field_values_arr([Value | Rest], TAcc, FAcc) ->
- Fields = get_text_field_values(Value, TAcc),
- get_text_field_values_arr(Rest, TAcc, Fields ++ FAcc).
-
-
-get_default_enabled(Props) ->
- case couch_util:get_value(<<"default_field">>, Props, {[]}) of
- Bool when is_boolean(Bool) ->
- Bool;
- {[]} ->
- true;
- {Opts}->
- couch_util:get_value(<<"enabled">>, Opts, true)
- end.
-
-
-get_index_array_lengths(Props) ->
- couch_util:get_value(<<"index_array_lengths">>, Props, true).
-
-
-add_default_text_field(Fields) ->
- DefaultFields = add_default_text_field(Fields, []),
- DefaultFields ++ Fields.
-
-
-add_default_text_field([], Acc) ->
- Acc;
-add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) ->
- NewAcc = [{<<"$default">>, <<"string">>, Value} | Acc],
- add_default_text_field(Rest, NewAcc);
-add_default_text_field([_ | Rest], Acc) ->
- add_default_text_field(Rest, Acc).
-
-
-%% index of all field names
-get_field_names(Fields) ->
- FieldNameSet = lists:foldl(fun({Name, _, _}, Set) ->
- gb_sets:add([<<"$fieldnames">>, Name, []], Set)
- end, gb_sets:new(), Fields),
- gb_sets:to_list(FieldNameSet).
-
-
-convert_text_fields([]) ->
- [];
-convert_text_fields([{Name, _Type, Value} | Rest]) ->
- [[Name, Value, []] | convert_text_fields(Rest)].
-
-
-should_index(Selector, Doc) ->
- % We should do this
- NormSelector = mango_selector:normalize(Selector),
- Matches = mango_selector:match(NormSelector, Doc),
- IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
- <<"_design/", _/binary>> -> true;
- _ -> false
- end,
- Matches and not IsDesign.
-
-
-get_text_field_list(IdxProps) ->
- case couch_util:get_value(<<"fields">>, IdxProps) of
- Fields when is_list(Fields) ->
- RawList = lists:flatmap(fun get_text_field_info/1, Fields),
- [mango_util:lucene_escape_user(Field) || Field <- RawList];
- _ ->
- all_fields
- end.
-
-
-get_text_field_info({Props}) ->
- Name = couch_util:get_value(<<"name">>, Props),
- Type0 = couch_util:get_value(<<"type">>, Props),
- if not is_binary(Name) -> []; true ->
- Type = get_text_field_type(Type0),
- [iolist_to_binary([Name, ":", Type])]
- end.
-
-
-get_text_field_type(<<"number">>) ->
- <<"number">>;
-get_text_field_type(<<"boolean">>) ->
- <<"boolean">>;
-get_text_field_type(_) ->
- <<"string">>.
-
-
-make_text_field(TAcc, Type, Value) ->
- FieldName = make_text_field_name(TAcc#tacc.path, Type),
- Fields = TAcc#tacc.fields,
- case Fields == all_fields orelse lists:member(FieldName, Fields) of
- true ->
- [{FieldName, Type, Value}];
- false ->
- []
- end.
-
-
-make_text_field_name([P | Rest], Type) ->
- Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
- Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
- iolist_to_binary(mango_util:join(".", Escaped)).
-
-
-validate_index_info(IndexInfo) ->
- IdxTypes = [mango_idx_view, mango_idx_text],
- Results = lists:foldl(fun(IdxType, Results0) ->
- try
- IdxType:validate_index_def(IndexInfo),
- [valid_index | Results0]
- catch _:_ ->
- [invalid_index | Results0]
- end
- end, [], IdxTypes),
- lists:member(valid_index, Results).
-
-
--ifdef(TEST).
-
--include_lib("eunit/include/eunit.hrl").
-
-handle_garbage_collect_cast_test() ->
- ?assertEqual({noreply, []}, handle_cast(garbage_collect, [])).
-
-handle_stop_cast_test() ->
- ?assertEqual({stop, normal, []}, handle_cast(stop, [])).
-
-handle_invalid_cast_test() ->
- ?assertEqual({stop, {invalid_cast, random}, []}, handle_cast(random, [])).
-
--endif.
[couchdb] 08/18: Add couch_views_encoding max value
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fd323c171e14771af2dfa889df3d99a60c01861d
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 24 15:48:14 2020 +0200
Add couch_views_encoding max value
Adds a max value to use for encoding. This is useful when getting the
max range when encoding startkey/endkeys.
---
src/couch_views/src/couch_views_encoding.erl | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/src/couch_views/src/couch_views_encoding.erl b/src/couch_views/src/couch_views_encoding.erl
index ef5fed9..2f69db3 100644
--- a/src/couch_views/src/couch_views_encoding.erl
+++ b/src/couch_views/src/couch_views_encoding.erl
@@ -14,6 +14,7 @@
-export([
+ max/0,
encode/1,
encode/2,
decode/1
@@ -27,6 +28,11 @@
-define(STRING, 4).
-define(LIST, 5).
-define(OBJECT, 6).
+-define(MAX, 255).
+
+
+max() ->
+ max_encoding_value.
encode(X) ->
@@ -51,6 +57,9 @@ encode_int(false, _Type) ->
encode_int(true, _Type) ->
{?TRUE};
+encode_int(max_encoding_value, _Type) ->
+ {?MAX};
+
encode_int(Num, key) when is_number(Num) ->
{?NUMBER, float(Num)};
@@ -87,6 +96,9 @@ decode_int({?FALSE}) ->
decode_int({?TRUE}) ->
true;
+decode_int({?MAX}) ->
+ max_encoding_value;
+
decode_int({?STRING, Bin}) ->
Bin;
[couchdb] 05/18: Add couch_views_indexer build to creation
versionstamp
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9dfe9e53532d8193ff5ee7a5482437879c95a851
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 14:28:37 2020 +0200
Add couch_views_indexer build to creation versionstamp
This creates a versionstamp for when an indexed was created
and build status for indexes. if the index has a creation_vs, then
couch_views_indexer will built the index to this creation versionstamp.
---
src/couch_views/include/couch_views.hrl | 6 ++
src/couch_views/src/couch_views_fdb.erl | 76 +++++++++++++++++++++
src/couch_views/src/couch_views_indexer.erl | 77 ++++++++++++++++------
src/couch_views/src/couch_views_jobs.erl | 21 ++++--
src/couch_views/test/couch_views_indexer_test.erl | 42 +++++++++++-
.../test/couch_views_trace_index_test.erl | 5 +-
src/fabric/src/fabric2_fdb.erl | 12 ++--
7 files changed, 206 insertions(+), 33 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index c40bb02..3d0110f 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -18,6 +18,8 @@
-define(VIEW_UPDATE_SEQ, 0).
-define(VIEW_ROW_COUNT, 1).
-define(VIEW_KV_SIZE, 2).
+-define(VIEW_BUILD_STATUS, 3).
+-define(VIEW_CREATION_VS, 4).
% Data keys
-define(VIEW_ID_RANGE, 0).
@@ -25,3 +27,7 @@
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
+
+% indexing progress
+-define(INDEX_BUILDING, <<"building">>).
+-define(INDEX_READY, <<"ready">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index a0224b2..3b008d4 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -13,6 +13,12 @@
-module(couch_views_fdb).
-export([
+ new_interactive_index/3,
+ new_creation_vs/3,
+ get_creation_vs/2,
+ get_build_status/2,
+ set_build_status/3,
+
get_update_seq/2,
set_update_seq/3,
@@ -39,6 +45,60 @@
-include_lib("fabric/include/fabric2.hrl").
+new_interactive_index(Db, Mrst, VS) ->
+ couch_views_fdb:new_creation_vs(Db, Mrst, VS),
+ couch_views_fdb:set_build_status(Db, Mrst, ?INDEX_BUILDING).
+
+
+%Interactive View Creation Versionstamp
+%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig) = VS
+
+new_creation_vs(TxDb, #mrst{} = Mrst, VS) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = creation_vs_key(TxDb, Mrst#mrst.sig),
+ Value = erlfdb_tuple:pack_vs({VS}),
+ ok = erlfdb:set_versionstamped_value(Tx, Key, Value).
+
+
+get_creation_vs(TxDb, #mrst{} = Mrst) ->
+ get_creation_vs(TxDb, Mrst#mrst.sig);
+
+get_creation_vs(TxDb, Sig) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = creation_vs_key(TxDb, Sig),
+ case erlfdb:wait(erlfdb:get(Tx, Key)) of
+ not_found ->
+ not_found;
+ EK ->
+ {VS} = erlfdb_tuple:unpack(EK),
+ VS
+ end.
+
+
+%Interactive View Build Status
+%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig) = INDEX_BUILDING | INDEX_READY
+
+get_build_status(TxDb, #mrst{sig = Sig}) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = build_status_key(TxDb, Sig),
+ erlfdb:wait(erlfdb:get(Tx, Key)).
+
+
+set_build_status(TxDb, #mrst{sig = Sig}, State) ->
+ #{
+ tx := Tx
+ } = TxDb,
+
+ Key = build_status_key(TxDb, Sig),
+ ok = erlfdb:set(Tx, Key, State).
+
+
% View Build Sequence Access
% (<db>, ?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ) = Sequence
@@ -340,6 +400,22 @@ map_idx_range(DbPrefix, Sig, ViewId, MapKey, DocId) ->
erlfdb_tuple:range(Key, DbPrefix).
+creation_vs_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+build_status_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
process_rows(Rows) ->
Encoded = lists:map(fun({K, V}) ->
EK1 = couch_views_encoding:encode(K, key),
diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index 0127bac..ab5aaad 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -18,7 +18,9 @@
-export([
- init/0
+ init/0,
+ map_docs/2,
+ write_docs/4
]).
-ifdef(TEST).
@@ -80,6 +82,7 @@ init() ->
db_seq => undefined,
view_seq => undefined,
last_seq => undefined,
+ view_vs => undefined,
job => Job,
job_data => Data,
count => 0,
@@ -174,22 +177,7 @@ update(#{} = Db, Mrst0, State0) ->
do_update(Db, Mrst0, State0) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
- % In the first iteration of update we need
- % to populate our db and view sequences
- State1 = case State0 of
- #{db_seq := undefined} ->
- ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst0),
- State0#{
- tx_db := TxDb,
- db_seq := fabric2_db:get_update_seq(TxDb),
- view_seq := ViewSeq,
- last_seq := ViewSeq
- };
- _ ->
- State0#{
- tx_db := TxDb
- }
- end,
+ State1 = get_update_start_state(TxDb, Mrst0, State0),
{ok, State2} = fold_changes(State1),
@@ -198,7 +186,8 @@ do_update(Db, Mrst0, State0) ->
doc_acc := DocAcc,
last_seq := LastSeq,
limit := Limit,
- limiter := Limiter
+ limiter := Limiter,
+ view_vs := ViewVS
} = State2,
DocAcc1 = fetch_docs(TxDb, DocAcc),
couch_rate:in(Limiter, Count),
@@ -210,6 +199,8 @@ do_update(Db, Mrst0, State0) ->
case Count < Limit of
true ->
+ maybe_set_build_status(TxDb, Mrst1, ViewVS,
+ ?INDEX_READY),
report_progress(State2, finished),
{Mrst1, finished};
false ->
@@ -224,6 +215,33 @@ do_update(Db, Mrst0, State0) ->
end).
+maybe_set_build_status(_TxDb, _Mrst1, not_found, _State) ->
+ ok;
+
+maybe_set_build_status(TxDb, Mrst1, _ViewVS, State) ->
+ couch_views_fdb:set_build_status(TxDb, Mrst1, State).
+
+
+% In the first iteration of update we need
+% to populate our db and view sequences
+get_update_start_state(TxDb, Mrst, #{db_seq := undefined} = State) ->
+ ViewVS = couch_views_fdb:get_creation_vs(TxDb, Mrst),
+ ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
+
+ State#{
+ tx_db := TxDb,
+ db_seq := fabric2_db:get_update_seq(TxDb),
+ view_vs := ViewVS,
+ view_seq := ViewSeq,
+ last_seq := ViewSeq
+ };
+
+get_update_start_state(TxDb, _Idx, State) ->
+ State#{
+ tx_db := TxDb
+ }.
+
+
fold_changes(State) ->
#{
view_seq := SinceSeq,
@@ -240,7 +258,8 @@ process_changes(Change, Acc) ->
#{
doc_acc := DocAcc,
count := Count,
- design_opts := DesignOpts
+ design_opts := DesignOpts,
+ view_vs := ViewVS
} = Acc,
#{
@@ -263,8 +282,22 @@ process_changes(Change, Acc) ->
last_seq := LastSeq
}
end,
- {ok, Acc1}.
+ DocVS = fabric2_fdb:seq_to_vs(LastSeq),
+
+ Go = maybe_stop_at_vs(ViewVS, DocVS),
+ {Go, Acc1}.
+
+
+maybe_stop_at_vs({versionstamp, _} = ViewVS, DocVS) when DocVS >= ViewVS ->
+ stop;
+
+maybe_stop_at_vs(_, _) ->
+ ok.
+
+
+map_docs(Mrst, []) ->
+ {Mrst, []};
map_docs(Mrst, Docs) ->
% Run all the non deleted docs through the view engine and
@@ -328,7 +361,9 @@ write_docs(TxDb, Mrst, Docs, State) ->
N + 1
end, 0, Docs),
- couch_views_fdb:set_update_seq(TxDb, Sig, LastSeq),
+ if LastSeq == false -> ok; true ->
+ couch_views_fdb:set_update_seq(TxDb, Sig, LastSeq)
+ end,
DocsNumber.
diff --git a/src/couch_views/src/couch_views_jobs.erl b/src/couch_views/src/couch_views_jobs.erl
index 1604841..b97e7ce 100644
--- a/src/couch_views/src/couch_views_jobs.erl
+++ b/src/couch_views/src/couch_views_jobs.erl
@@ -40,11 +40,12 @@ build_view(TxDb, Mrst, UpdateSeq) ->
end.
-build_view_async(TxDb, Mrst) ->
- JobId = job_id(TxDb, Mrst),
- JobData = job_data(TxDb, Mrst),
- DbUUID = fabric2_db:get_uuid(TxDb),
- couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(), fun(JTx) ->
+build_view_async(TxDb0, Mrst) ->
+ JobId = job_id(TxDb0, Mrst),
+ JobData = job_data(TxDb0, Mrst),
+ DbUUID = fabric2_db:get_uuid(TxDb0),
+ TxDb1 = ensure_correct_tx(TxDb0),
+ couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(TxDb1), fun(JTx) ->
case couch_jobs:get_job_data(JTx, ?INDEX_JOB_TYPE, JobId) of
{error, not_found} ->
ok;
@@ -59,6 +60,16 @@ build_view_async(TxDb, Mrst) ->
{ok, JobId}.
+ensure_correct_tx(#{tx := undefined} = TxDb) ->
+ TxDb;
+
+ensure_correct_tx(#{tx := Tx} = TxDb) ->
+ case erlfdb:is_read_only(Tx) of
+ true -> TxDb#{tx := undefined};
+ false -> TxDb
+ end.
+
+
wait_for_job(JobId, UpdateSeq) ->
case couch_jobs:subscribe(?INDEX_JOB_TYPE, JobId) of
{ok, Subscription, _State, _Data} ->
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index 43b5828..8ddb64b 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -51,7 +51,8 @@ indexer_test_() ->
?TDEF_FE(index_autoupdater_callback),
?TDEF_FE(handle_db_recreated_when_running),
?TDEF_FE(handle_db_recreated_after_finished),
- ?TDEF_FE(index_budget_is_changing)
+ ?TDEF_FE(index_budget_is_changing),
+ ?TDEF_FE(index_can_recover_from_crash, 60)
]
}
}
@@ -508,6 +509,41 @@ handle_db_recreated_after_finished(Db) ->
], Out2).
+index_can_recover_from_crash(Db) ->
+ ok = meck:new(config, [passthrough]),
+ ok = meck:expect(config, get_integer, fun(Section, Key, Default) ->
+ case Section == "couch_views" andalso Key == "change_limit" of
+ true -> 1;
+ _ -> Default
+ end
+ end),
+ meck:new(couch_eval, [passthrough]),
+ meck:expect(couch_eval, map_docs, fun(State, Docs) ->
+ Doc = hd(Docs),
+ case Doc#doc.id == <<"2">> of
+ true ->
+ % remove the mock so that next time the doc is processed
+ % it will work
+ meck:unload(couch_eval),
+ throw({fake_crash, test_jobs_restart});
+ false ->
+ meck:passthrough([State, Docs])
+ end
+ end),
+
+ DDoc = create_ddoc(),
+ Docs = make_docs(3),
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_docs(Db, Docs, []),
+
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([
+ row(<<"1">>, 1, 1),
+ row(<<"2">>, 2, 2),
+ row(<<"3">>, 3, 3)
+ ], Out).
+
+
row(Id, Key, Value) ->
{row, [
{id, Id},
@@ -603,6 +639,10 @@ create_ddoc(multi_emit_key_limit) ->
]}).
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
doc(Id) ->
doc(Id, Id).
diff --git a/src/couch_views/test/couch_views_trace_index_test.erl b/src/couch_views/test/couch_views_trace_index_test.erl
index f8a5ce5..5b15a4c 100644
--- a/src/couch_views/test/couch_views_trace_index_test.erl
+++ b/src/couch_views/test/couch_views_trace_index_test.erl
@@ -77,10 +77,13 @@ trace_single_doc(Db) ->
{ok, _} = fabric2_db:update_doc(Db, Doc, []),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+ HexSig = fabric2_util:to_hex(Mrst#mrst.sig),
JobData = #{
<<"db_name">> => DbName,
+ <<"db_uuid">> => fabric2_db:get_uuid(Db),
<<"ddoc_id">> => <<"_design/bar">>,
- <<"sig">> => fabric2_util:to_hex(Mrst#mrst.sig)
+ <<"sig">> => HexSig,
+ <<"retries">> => 0
},
meck:expect(couch_jobs, accept, 2, {ok, job, JobData}),
meck:expect(couch_jobs, update, 3, {ok, job}),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 912d4df..2295a56 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -64,6 +64,8 @@
seq_to_vs/1,
next_vs/1,
+ new_versionstamp/1,
+
debug_cluster/0,
debug_cluster/2
]).
@@ -1021,6 +1023,11 @@ next_vs({versionstamp, VS, Batch, TxId}) ->
{versionstamp, V, B, T}.
+new_versionstamp(Tx) ->
+ TxId = erlfdb:get_next_tx_id(Tx),
+ {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
+
+
debug_cluster() ->
debug_cluster(<<>>, <<16#FE, 16#FF, 16#FF>>).
@@ -1763,11 +1770,6 @@ get_transaction_id(Tx, LayerPrefix) ->
end.
-new_versionstamp(Tx) ->
- TxId = erlfdb:get_next_tx_id(Tx),
- {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
-
-
on_commit(Tx, Fun) when is_function(Fun, 0) ->
% Here we rely on Tx objects matching. However they contain a nif resource
% object. Before Erlang 20.0 those would have been represented as empty
[couchdb] 02/18: move all_doc view options to fabric2_util
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3179147448de78ad1a130ba9f6ef4c24044cad95
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 18 09:40:24 2020 +0200
move all_doc view options to fabric2_util
---
src/chttpd/src/chttpd_db.erl | 30 +-----------------------------
src/fabric/src/fabric2_util.erl | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 29 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 174d464..e9b33f0 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -867,7 +867,7 @@ all_docs_view(Req, Db, Keys, OP) ->
send_all_docs(Db, #mrargs{keys = undefined} = Args, VAcc0) ->
- Opts0 = all_docs_view_opts(Args),
+ Opts0 = fabric2_util:all_docs_view_opts(Args),
Opts = Opts0 ++ [{restart_tx, true}],
NS = couch_util:get_value(namespace, Opts),
FoldFun = case NS of
@@ -939,34 +939,6 @@ send_all_docs_keys(Db, #mrargs{} = Args, VAcc0) ->
end, VAcc1, Keys).
-all_docs_view_opts(Args) ->
- NS = couch_util:get_value(namespace, Args#mrargs.extra),
- StartKey = case Args#mrargs.start_key of
- undefined -> Args#mrargs.start_key_docid;
- SKey -> SKey
- end,
- EndKey = case Args#mrargs.end_key of
- undefined -> Args#mrargs.end_key_docid;
- EKey -> EKey
- end,
- StartKeyOpts = case StartKey of
- undefined -> [];
- _ -> [{start_key, fabric2_util:encode_all_doc_key(StartKey)}]
- end,
- EndKeyOpts = case {EndKey, Args#mrargs.inclusive_end} of
- {undefined, _} -> [];
- {_, false} -> [{end_key_gt, fabric2_util:encode_all_doc_key(EndKey)}];
- {_, true} -> [{end_key, fabric2_util:encode_all_doc_key(EndKey)}]
- end,
- [
- {dir, Args#mrargs.direction},
- {limit, Args#mrargs.limit},
- {skip, Args#mrargs.skip},
- {update_seq, Args#mrargs.update_seq},
- {namespace, NS}
- ] ++ StartKeyOpts ++ EndKeyOpts.
-
-
apply_args_to_keylist(Args, Keys0) ->
Keys1 = case Args#mrargs.direction of
fwd -> Keys0;
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index d74ef27..97bfedc 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -38,6 +38,7 @@
uuid/0,
encode_all_doc_key/1,
+ all_docs_view_opts/1,
pmap/2,
pmap/3
@@ -45,6 +46,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
revinfo_to_revs(RevInfo) ->
@@ -299,6 +301,42 @@ encode_all_doc_key(Term) when Term < <<>> -> <<>>;
encode_all_doc_key(_) -> <<255>>.
+all_docs_view_opts(#mrargs{} = Args) ->
+ NS = couch_util:get_value(namespace, Args#mrargs.extra),
+ StartKey = case Args#mrargs.start_key of
+ undefined -> Args#mrargs.start_key_docid;
+ SKey -> SKey
+ end,
+ EndKey = case Args#mrargs.end_key of
+ undefined -> Args#mrargs.end_key_docid;
+ EKey -> EKey
+ end,
+ StartKeyOpts = case StartKey of
+ undefined -> [];
+ _ -> [{start_key, encode_all_doc_key(StartKey)}]
+ end,
+ EndKeyOpts = case {EndKey, Args#mrargs.inclusive_end} of
+ {undefined, _} -> [];
+ {_, false} -> [{end_key_gt, encode_all_doc_key(EndKey)}];
+ {_, true} -> [{end_key, encode_all_doc_key(EndKey)}]
+ end,
+
+ DocOpts = case Args#mrargs.conflicts of
+ true -> [conflicts | Args#mrargs.doc_options];
+ _ -> Args#mrargs.doc_options
+ end,
+
+ [
+ {dir, Args#mrargs.direction},
+ {limit, Args#mrargs.limit},
+ {skip, Args#mrargs.skip},
+ {update_seq, Args#mrargs.update_seq},
+ {namespace, NS},
+ {include_docs, Args#mrargs.include_docs},
+ {doc_opts, DocOpts}
+ ] ++ StartKeyOpts ++ EndKeyOpts.
+
+
pmap(Fun, Args) ->
pmap(Fun, Args, []).