You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2020/02/26 06:19:08 UTC
[couchdb] branch fdb-mango-indexes-without-common-changes created
(now 86c69c7)
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a change to branch fdb-mango-indexes-without-common-changes
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 86c69c7 Mango Indexes on FoundationDB
This branch includes the following new commits:
new 86c69c7 Mango Indexes on FoundationDB
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Mango Indexes on FoundationDB
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch fdb-mango-indexes-without-common-changes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 86c69c72eda56a660daaa276e39a188c9c54382c
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Feb 25 22:18:30 2020 -0800
Mango Indexes on FoundationDB
Updates Mango view indexes to work on top of FoundationDB.
Mango view indexes are replaced by Mango json indexes.
Mango json indexes are updated in the same transaction that a document
is updated. A background indexer will build a new index and get it up
to date.
contributor: Jay Doane
(these have changes to README, TODO, and rebar.config files removed so
they can be more easily added to maintenance branches)
---
src/couch/src/couch_proc_manager.erl | 2 +-
src/couch_js/src/couch_js_proc_manager.erl | 2 +-
src/couch_views/src/couch_views_indexer.erl | 8 +-
src/couch_views/src/couch_views_server.erl | 19 +-
src/couch_views/src/couch_views_sup.erl | 3 +-
src/fabric/include/fabric2.hrl | 1 +
src/fabric/src/fabric2_db.erl | 28 +-
src/fabric/src/fabric2_fdb.erl | 51 ++-
src/mango/src/mango.hrl | 9 +
src/mango/src/mango_crud.erl | 35 +-
src/mango/src/mango_cursor.erl | 7 +
src/mango/src/mango_cursor_view.erl | 387 +++++------------------
src/mango/src/mango_execution_stats.erl | 8 -
src/mango/src/mango_execution_stats.hrl | 1 -
src/mango/src/mango_fdb.erl | 184 +++++++++++
src/mango/src/mango_fdb_special.erl | 86 +++++
src/mango/src/mango_fdb_view.erl | 101 ++++++
src/mango/src/mango_httpd.erl | 12 +-
src/mango/src/mango_idx.erl | 135 ++++++--
src/mango/src/mango_idx.hrl | 3 +-
src/mango/src/mango_idx_special.erl | 4 +-
src/mango/src/mango_idx_view.erl | 5 +-
src/mango/src/mango_idx_view.hrl | 2 +-
src/mango/src/mango_indexer.erl | 172 ++++++++++
src/mango/src/mango_jobs.erl | 50 +++
src/mango/src/mango_jobs_indexer.erl | 325 +++++++++++++++++++
src/mango/src/mango_json_bookmark.erl | 24 +-
src/mango/src/mango_native_proc.erl | 3 +-
src/mango/src/mango_sup.erl | 15 +-
src/mango/src/mango_util.erl | 11 +-
src/mango/test/01-index-crud-test.py | 20 +-
src/mango/test/02-basic-find-test.py | 12 +-
src/mango/test/03-operator-test.py | 4 +-
src/mango/test/05-index-selection-test.py | 1 +
src/mango/test/11-ignore-design-docs-test.py | 2 +-
src/mango/test/12-use-correct-index-test.py | 40 ++-
src/mango/test/13-stable-update-test.py | 2 +
src/mango/test/13-users-db-find-test.py | 118 +++----
src/mango/test/15-execution-stats-test.py | 5 +-
src/mango/test/16-index-selectors-test.py | 2 +
src/mango/test/17-multi-type-value-test.py | 4 +-
src/mango/test/18-json-sort.py | 6 +-
src/mango/test/19-find-conflicts.py | 3 +-
src/mango/test/20-no-timeout-test.py | 15 +-
src/mango/test/eunit/mango_indexer_test.erl | 164 ++++++++++
src/mango/test/eunit/mango_jobs_indexer_test.erl | 189 +++++++++++
src/mango/test/mango.py | 27 +-
src/mango/test/user_docs.py | 6 +-
48 files changed, 1777 insertions(+), 536 deletions(-)
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 3366b2b..0f686ef 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl
index 0964696..0f2916b 100644
--- a/src/couch_js/src/couch_js_proc_manager.erl
+++ b/src/couch_js/src/couch_js_proc_manager.erl
@@ -108,7 +108,7 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+%% ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index 31cd8e6..29e3920 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -18,7 +18,9 @@
-export([
- init/0
+ set_timeout/0,
+ init/0,
+ fetch_docs/2
]).
-include("couch_views.hrl").
@@ -34,6 +36,10 @@ spawn_link() ->
proc_lib:spawn_link(?MODULE, init, []).
+set_timeout() ->
+ couch_views_jobs:set_timeout().
+
+
init() ->
{ok, Job, Data0} = couch_jobs:accept(?INDEX_JOB_TYPE, #{}),
Data = upgrade_data(Data0),
diff --git a/src/couch_views/src/couch_views_server.erl b/src/couch_views/src/couch_views_server.erl
index d14216e..b07c85f 100644
--- a/src/couch_views/src/couch_views_server.erl
+++ b/src/couch_views/src/couch_views_server.erl
@@ -17,7 +17,7 @@
-export([
- start_link/0
+ start_link/1
]).
@@ -34,16 +34,18 @@
-define(MAX_WORKERS, 100).
-start_link() ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+start_link(Opts) ->
+ gen_server:start_link(?MODULE, Opts, []).
-init(_) ->
+init(Opts) ->
+ WorkerModule = couch_util:get_value(worker, Opts, couch_views_indexer),
process_flag(trap_exit, true),
- couch_views_jobs:set_timeout(),
+ WorkerModule:set_timeout(),
St = #{
workers => #{},
- max_workers => max_workers()
+ max_workers => max_workers(),
+ worker_module => WorkerModule
},
{ok, spawn_workers(St)}.
@@ -87,11 +89,12 @@ code_change(_OldVsn, St, _Extra) ->
spawn_workers(St) ->
#{
workers := Workers,
- max_workers := MaxWorkers
+ max_workers := MaxWorkers,
+ worker_module := WorkerModule
} = St,
case maps:size(Workers) < MaxWorkers of
true ->
- Pid = couch_views_indexer:spawn_link(),
+ Pid = WorkerModule:spawn_link(),
NewSt = St#{workers := Workers#{Pid => true}},
spawn_workers(NewSt);
false ->
diff --git a/src/couch_views/src/couch_views_sup.erl b/src/couch_views/src/couch_views_sup.erl
index 7a72a1f..c3256fd 100644
--- a/src/couch_views/src/couch_views_sup.erl
+++ b/src/couch_views/src/couch_views_sup.erl
@@ -36,10 +36,11 @@ start_link() ->
init(normal) ->
+ Args = [{worker, couch_views_indexer}],
Children = [
#{
id => couch_views_server,
- start => {couch_views_server, start_link, []}
+ start => {couch_views_server, start_link, [Args]}
}
],
{ok, {flags(), Children}};
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index f526d7b..cc566aa 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -39,6 +39,7 @@
-define(DB_LOCAL_DOC_BODIES, 25).
-define(DB_ATT_NAMES, 26).
-define(DB_SEARCH, 27).
+-define(DB_MANGO, 28).
% Versions
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index b0f7849..eeaf3f1 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -75,6 +75,7 @@
open_doc/2,
open_doc/3,
open_doc_revs/4,
+ apply_open_doc_opts/3,
%% open_doc_int/3,
get_doc_info/2,
get_full_doc_info/2,
@@ -799,11 +800,23 @@ fold_docs(Db, UserFun, UserAcc0, Options) ->
UserAcc2 = fabric2_fdb:fold_range(TxDb, Prefix, fun({K, V}, Acc) ->
{DocId} = erlfdb_tuple:unpack(K, Prefix),
RevId = erlfdb_tuple:unpack(V),
- maybe_stop(UserFun({row, [
+ Row0 = [
{id, DocId},
{key, DocId},
{value, {[{rev, couch_doc:rev_to_str(RevId)}]}}
- ]}, Acc))
+ ],
+
+ DocOpts = couch_util:get_value(doc_opts, Options, []),
+ OpenOpts = [deleted | DocOpts],
+
+ Row1 = case lists:keyfind(include_docs, 1, Options) of
+ {include_docs, true} ->
+ Row0 ++ open_json_doc(Db, DocId, OpenOpts, DocOpts);
+ _ ->
+ Row0
+ end,
+
+ maybe_stop(UserFun({row, Row1}, Acc))
end, UserAcc1, Options),
{ok, maybe_stop(UserFun(complete, UserAcc2))}
@@ -1845,3 +1858,14 @@ stem_revisions(#{} = Db, #doc{} = Doc) ->
true -> Doc#doc{revs = {RevPos, lists:sublist(Revs, RevsLimit)}};
false -> Doc
end.
+
+
+open_json_doc(Db, DocId, OpenOpts, DocOpts) ->
+ case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ [];
+ {ok, #doc{deleted = true}} ->
+ [{doc, null}];
+ {ok, #doc{} = Doc} ->
+ [{doc, couch_doc:to_json_obj(Doc, DocOpts)}]
+ end.
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index c34b33c..5e3d70f 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -64,6 +64,8 @@
seq_to_vs/1,
next_vs/1,
+ new_versionstamp/1,
+
debug_cluster/0,
debug_cluster/2
]).
@@ -647,8 +649,12 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
atts = Atts
} = Doc,
- % Doc body
+ % Fetch the old doc body for the mango hooks
+ OldWinnerDoc = if OldWinner == not_found -> not_found; true ->
+ get_doc_body(Db, DocId, OldWinner)
+ end,
+ % Doc body
ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -761,13 +767,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
- incr_stat(Db, <<"doc_count">>, 1);
+ incr_stat(Db, <<"doc_count">>, 1),
+ mango_indexer:create_doc(Db, Doc);
recreated ->
if not IsDDoc -> ok; true ->
incr_stat(Db, <<"doc_design_count">>, 1)
end,
incr_stat(Db, <<"doc_count">>, 1),
- incr_stat(Db, <<"doc_del_count">>, -1);
+ incr_stat(Db, <<"doc_del_count">>, -1),
+ mango_indexer:create_doc(Db, Doc);
replicate_deleted ->
incr_stat(Db, <<"doc_del_count">>, 1);
ignore ->
@@ -777,9 +785,29 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
incr_stat(Db, <<"doc_design_count">>, -1)
end,
incr_stat(Db, <<"doc_count">>, -1),
- incr_stat(Db, <<"doc_del_count">>, 1);
+ incr_stat(Db, <<"doc_del_count">>, 1),
+ mango_indexer:delete_doc(Db, OldWinnerDoc);
updated ->
- ok
+ % Get winning doc with conflicts field
+ DocRev = extract_rev(Doc#doc.revs),
+ {WinnerRevPos, _} = WinnerRevId = maps:get(rev_id, NewWinner),
+ {WinnerDoc, OldWinnerDoc} = case WinnerRevId == DocRev of
+ true -> {Doc, OldWinnerDoc};
+ false -> {OldWinnerDoc, OldWinnerDoc}
+ end,
+
+ RevConflicts = lists:foldl(fun (UpdateRev, Acc) ->
+ {RevPos, _} = maps:get(rev_id, UpdateRev),
+ case RevPos == WinnerRevPos of
+ true ->
+ Acc ++ [UpdateRev#{winner := false}];
+ false ->
+ Acc
+ end
+ end, [], ToUpdate),
+
+ {ok, WinnerDoc1} = fabric2_db:apply_open_doc_opts(WinnerDoc, RevConflicts, [conflicts]),
+ mango_indexer:update_doc(Db, WinnerDoc1, OldWinnerDoc)
end,
% Update database size
@@ -789,6 +817,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok.
+extract_rev({RevPos, [Rev | _]}) ->
+ {RevPos, Rev}.
+
write_local_doc(#{} = Db0, Doc) ->
#{
@@ -968,6 +999,11 @@ next_vs({versionstamp, VS, Batch, TxId}) ->
{versionstamp, V, B, T}.
+new_versionstamp(Tx) ->
+ TxId = erlfdb:get_next_tx_id(Tx),
+ {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
+
+
debug_cluster() ->
debug_cluster(<<>>, <<16#FE, 16#FF, 16#FF>>).
@@ -1704,11 +1740,6 @@ get_transaction_id(Tx, LayerPrefix) ->
end.
-new_versionstamp(Tx) ->
- TxId = erlfdb:get_next_tx_id(Tx),
- {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
-
-
on_commit(Tx, Fun) when is_function(Fun, 0) ->
% Here we rely on Tx objects matching. However they contain a nif resource
% object. Before Erlang 20.0 those would have been represented as empty
diff --git a/src/mango/src/mango.hrl b/src/mango/src/mango.hrl
index 26a9d43..a1f9325 100644
--- a/src/mango/src/mango.hrl
+++ b/src/mango/src/mango.hrl
@@ -11,3 +11,12 @@
% the License.
-define(MANGO_ERROR(R), throw({mango_error, ?MODULE, R})).
+
+-define(MANGO_IDX_BUILD_STATUS, 1).
+-define(MANGO_UPDATE_SEQ, 2).
+-define(MANGO_IDX_RANGE, 3).
+
+-define(MANGO_INDEX_JOB_TYPE, <<"mango">>).
+
+-define(MANGO_INDEX_BUILDING, <<"building">>).
+-define(MANGO_INDEX_READY, <<"ready">>).
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 41a4d14..66cef65 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -33,10 +33,8 @@ insert(Db, #doc{}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
-insert(Db, Docs, Opts0) when is_list(Docs) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(w, Opts1),
- case fabric:update_docs(Db, Docs, Opts2) of
+insert(Db, Docs, Opts) when is_list(Docs) ->
+ case fabric2_db:update_docs(Db, Docs, Opts) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -46,10 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
end.
-find(Db, Selector, Callback, UserAcc, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+find(Db, Selector, Callback, UserAcc, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -99,30 +95,11 @@ delete(Db, Selector, Options) ->
end.
-explain(Db, Selector, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+explain(Db, Selector, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:explain(Cursor).
-maybe_add_user_ctx(Db, Opts) ->
- case lists:keyfind(user_ctx, 1, Opts) of
- {user_ctx, _} ->
- Opts;
- false ->
- [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
- end.
-
-
-maybe_int_to_str(_Key, []) ->
- [];
-maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) ->
- [{Key, integer_to_list(Val)} | maybe_int_to_str(Key, Rest)];
-maybe_int_to_str(Key, [KV | Rest]) ->
- [KV | maybe_int_to_str(Key, Rest)].
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index c6f21dd..8bdf022 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,6 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
+ remove_unbuilt_indexes/1,
maybe_add_warning/3
]).
@@ -123,6 +124,12 @@ remove_indexes_with_partial_filter_selector(Indexes) ->
lists:filter(FiltFun, Indexes).
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, Indexes).
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 1c4b342..21c402b 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,9 +19,7 @@
]).
-export([
- view_cb/2,
handle_message/2,
- handle_all_docs_message/2,
composite_indexes/2,
choose_best_index/2
]).
@@ -44,7 +42,7 @@ create(Db, Indexes, Selector, Opts) ->
Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
Skip = couch_util:get_value(skip, Opts, 0),
Fields = couch_util:get_value(fields, Opts, all_fields),
- Bookmark = couch_util:get_value(bookmark, Opts),
+ Bookmark = couch_util:get_value(bookmark, Opts),
{ok, #cursor{
db = Db,
@@ -60,24 +58,23 @@ create(Db, Indexes, Selector, Opts) ->
explain(Cursor) ->
- #cursor{
- opts = Opts
- } = Cursor,
-
- BaseArgs = base_args(Cursor),
- Args = apply_opts(Opts, BaseArgs),
-
- [{mrargs, {[
- {include_docs, Args#mrargs.include_docs},
- {view_type, Args#mrargs.view_type},
- {reduce, Args#mrargs.reduce},
- {partition, couch_mrview_util:get_extra(Args, partition, null)},
- {start_key, maybe_replace_max_json(Args#mrargs.start_key)},
- {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
- {direction, Args#mrargs.direction},
- {stable, Args#mrargs.stable},
- {update, Args#mrargs.update},
- {conflicts, Args#mrargs.conflicts}
+ #{
+ start_key := StartKey,
+ end_key := EndKey,
+ dir := Direction
+ } = index_args(Cursor),
+
+ [{args, {[
+ {include_docs, true},
+ {view_type, <<"fdb">>},
+ {reduce, false},
+ {partition, false},
+ {start_key, maybe_replace_max_json(StartKey)},
+ {end_key, maybe_replace_max_json(EndKey)},
+ {direction, Direction},
+ {stable, false},
+ {update, true},
+ {conflicts, false}
]}}].
@@ -99,18 +96,47 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
- #mrargs{
- view_type = map,
- reduce = false,
- start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges),
- end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
- include_docs = true,
- extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
- }.
+%% TODO: When supported, handle:
+%% partitions
+%% conflicts
+index_args(#cursor{} = Cursor) ->
+ #cursor{
+ index = Idx,
+ opts = Opts,
+ bookmark = Bookmark
+ } = Cursor,
+
+ Args0 = #{
+ start_key => mango_idx:start_key(Idx, Cursor#cursor.ranges),
+ start_key_docid => <<>>,
+ end_key => mango_idx:end_key(Idx, Cursor#cursor.ranges),
+ end_key_docid => <<255>>,
+ skip => 0
+ },
+
+ Sort = couch_util:get_value(sort, Opts, [<<"asc">>]),
+ Args1 = case mango_sort:directions(Sort) of
+ [<<"desc">> | _] ->
+ #{
+ start_key := SK,
+ start_key_docid := SKDI,
+ end_key := EK,
+ end_key_docid := EKDI
+ } = Args0,
+ Args0#{
+ dir => rev,
+ start_key => EK,
+ start_key_docid => EKDI,
+ end_key => SK,
+ end_key_docid => SKDI
+ };
+ _ ->
+ Args0#{dir => fwd}
+ end,
+ mango_json_bookmark:update_args(Bookmark, Args1).
-execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
+execute(#cursor{db = Db, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
Cursor = Cursor0#cursor{
user_fun = UserFun,
user_acc = UserAcc,
@@ -121,32 +147,22 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
% empty indicates unsatisfiable ranges, so don't perform search
{ok, UserAcc};
_ ->
- BaseArgs = base_args(Cursor),
- #cursor{opts = Opts, bookmark = Bookmark} = Cursor,
- Args0 = apply_opts(Opts, BaseArgs),
- Args = mango_json_bookmark:update_args(Bookmark, Args0),
- UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
- DbOpts = [{user_ctx, UserCtx}],
- Result = case mango_idx:def(Idx) of
- all_docs ->
- CB = fun ?MODULE:handle_all_docs_message/2,
- fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
- _ ->
- CB = fun ?MODULE:handle_message/2,
- % Normal view
- DDoc = ddocid(Idx),
- Name = mango_idx:name(Idx),
- fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args)
- end,
+ Args = index_args(Cursor),
+ CB = fun ?MODULE:handle_message/2,
+ Result = mango_fdb:query(Db, CB, Cursor, Args),
case Result of
{ok, LastCursor} ->
NewBookmark = mango_json_bookmark:create(LastCursor),
Arg = {add_key, bookmark, NewBookmark},
- {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
- Stats0 = LastCursor#cursor.execution_stats,
- FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
- {ok, FinalUserAcc1};
+ #cursor{
+ opts = Opts,
+ execution_stats = Stats0,
+ user_acc = FinalUserAcc0
+ } = LastCursor,
+ {_Go, FinalUserAcc1} = UserFun(Arg, FinalUserAcc0),
+ FinalUserAcc2 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc1),
+ FinalUserAcc3 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc2),
+ {ok, FinalUserAcc3};
{error, Reason} ->
{error, Reason}
end
@@ -217,80 +233,16 @@ choose_best_index(_DbName, IndexRanges) ->
{SelectedIndex, SelectedIndexRanges}.
-view_cb({meta, Meta}, Acc) ->
- % Map function starting
- put(mango_docs_examined, 0),
- set_mango_msg_timestamp(),
- ok = rexi:stream2({meta, Meta}),
- {ok, Acc};
-view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
- ViewRow = #view_row{
- id = couch_util:get_value(id, Row),
- key = couch_util:get_value(key, Row),
- doc = couch_util:get_value(doc, Row)
- },
- case ViewRow#view_row.doc of
- null ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
- maybe_send_mango_ping();
- undefined ->
- ViewRow2 = ViewRow#view_row{
- value = couch_util:get_value(value, Row)
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
- set_mango_msg_timestamp();
- Doc ->
- Selector = couch_util:get_value(selector, Options),
- case mango_selector:match(Selector, Doc) of
- true ->
- ViewRow2 = ViewRow#view_row{
- value = get(mango_docs_examined) + 1
- },
- ok = rexi:stream2(ViewRow2),
- put(mango_docs_examined, 0),
- set_mango_msg_timestamp();
- false ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
- maybe_send_mango_ping()
- end
- end,
- {ok, Acc};
-view_cb(complete, Acc) ->
- % Finish view output
- ok = rexi:stream_last(complete),
- {ok, Acc};
-view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
-
-
-maybe_send_mango_ping() ->
- Current = os:timestamp(),
- LastPing = get(mango_last_msg_timestamp),
- % Fabric will timeout if it has not heard a response from a worker node
- % after 5 seconds. Send a ping every 4 seconds so the timeout doesn't happen.
- case timer:now_diff(Current, LastPing) > ?HEARTBEAT_INTERVAL_IN_USEC of
- false ->
- ok;
- true ->
- rexi:ping(),
- set_mango_msg_timestamp()
- end.
-
-
-set_mango_msg_timestamp() ->
- put(mango_last_msg_timestamp, os:timestamp()).
-
-
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
-handle_message({row, Props}, Cursor) ->
- case doc_member(Cursor, Props) of
+handle_message({doc, Key, Doc}, Cursor) ->
+ case match_doc(Cursor, Doc) of
{ok, Doc, {execution_stats, ExecutionStats1}} ->
Cursor1 = Cursor#cursor {
execution_stats = ExecutionStats1
},
- Cursor2 = update_bookmark_keys(Cursor1, Props),
+ {Props} = Doc,
+ Cursor2 = update_bookmark_keys(Cursor1, {Key, Props}),
FinalDoc = mango_fields:extract(Doc, Cursor2#cursor.fields),
handle_doc(Cursor2, FinalDoc);
{no_match, _, {execution_stats, ExecutionStats1}} ->
@@ -308,15 +260,6 @@ handle_message({error, Reason}, _Cursor) ->
{error, Reason}.
-handle_all_docs_message({row, Props}, Cursor) ->
- case is_design_doc(Props) of
- true -> {ok, Cursor};
- false -> handle_message({row, Props}, Cursor)
- end;
-handle_all_docs_message(Message, Cursor) ->
- handle_message(Message, Cursor).
-
-
handle_doc(#cursor{skip = S} = C, _) when S > 0 ->
{ok, C#cursor{skip = S - 1}};
handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 ->
@@ -332,191 +275,25 @@ handle_doc(C, _Doc) ->
{stop, C}.
-ddocid(Idx) ->
- case mango_idx:ddoc(Idx) of
- <<"_design/", Rest/binary>> ->
- Rest;
- Else ->
- Else
- end.
-
-
-apply_opts([], Args) ->
- Args;
-apply_opts([{r, RStr} | Rest], Args) ->
- IncludeDocs = case list_to_integer(RStr) of
- 1 ->
- true;
- R when R > 1 ->
- % We don't load the doc in the view query because
- % we have to do a quorum read in the coordinator
- % so there's no point.
- false
- end,
- NewArgs = Args#mrargs{include_docs = IncludeDocs},
- apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, true} | Rest], Args) ->
- NewArgs = Args#mrargs{conflicts = true},
- apply_opts(Rest, NewArgs);
-apply_opts([{conflicts, false} | Rest], Args) ->
- % Ignored cause default
- apply_opts(Rest, Args);
-apply_opts([{sort, Sort} | Rest], Args) ->
- % We only support single direction sorts
- % so nothing fancy here.
- case mango_sort:directions(Sort) of
- [] ->
- apply_opts(Rest, Args);
- [<<"asc">> | _] ->
- apply_opts(Rest, Args);
- [<<"desc">> | _] ->
- SK = Args#mrargs.start_key,
- SKDI = Args#mrargs.start_key_docid,
- EK = Args#mrargs.end_key,
- EKDI = Args#mrargs.end_key_docid,
- NewArgs = Args#mrargs{
- direction = rev,
- start_key = EK,
- start_key_docid = EKDI,
- end_key = SK,
- end_key_docid = SKDI
- },
- apply_opts(Rest, NewArgs)
- end;
-apply_opts([{stale, ok} | Rest], Args) ->
- NewArgs = Args#mrargs{
- stable = true,
- update = false
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{stable, true} | Rest], Args) ->
- NewArgs = Args#mrargs{
- stable = true
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{update, false} | Rest], Args) ->
- NewArgs = Args#mrargs{
- update = false
- },
- apply_opts(Rest, NewArgs);
-apply_opts([{partition, <<>>} | Rest], Args) ->
- apply_opts(Rest, Args);
-apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
- NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
- apply_opts(Rest, NewArgs);
-apply_opts([{_, _} | Rest], Args) ->
- % Ignore unknown options
- apply_opts(Rest, Args).
-
-
-doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
- Opts = Cursor#cursor.opts,
- ExecutionStats = Cursor#cursor.execution_stats,
+match_doc(Cursor, Doc) ->
+ ExecStats = Cursor#cursor.execution_stats,
Selector = Cursor#cursor.selector,
- {Matched, Incr} = case couch_util:get_value(value, RowProps) of
- N when is_integer(N) -> {true, N};
- _ -> {false, 1}
- end,
- case couch_util:get_value(doc, RowProps) of
- {DocProps} ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats, Incr),
- case Matched of
- true ->
- {ok, {DocProps}, {execution_stats, ExecutionStats1}};
- false ->
- match_doc(Selector, {DocProps}, ExecutionStats1)
- end;
- undefined ->
- ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
- Id = couch_util:get_value(id, RowProps),
- case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
- {ok, #doc{}=DocProps} ->
- Doc = couch_doc:to_json_obj(DocProps, []),
- match_doc(Selector, Doc, ExecutionStats1);
- Else ->
- Else
- end;
- null ->
- ExecutionStats1 = mango_execution_stats:incr_docs_examined(ExecutionStats),
- {no_match, null, {execution_stats, ExecutionStats1}}
- end.
-
+ ExecStats1 = mango_execution_stats:incr_docs_examined(ExecStats, 1),
-match_doc(Selector, Doc, ExecutionStats) ->
case mango_selector:match(Selector, Doc) of
true ->
- {ok, Doc, {execution_stats, ExecutionStats}};
+ {ok, Doc, {execution_stats, ExecStats1}};
false ->
- {no_match, Doc, {execution_stats, ExecutionStats}}
- end.
-
-
-is_design_doc(RowProps) ->
- case couch_util:get_value(id, RowProps) of
- <<"_design/", _/binary>> -> true;
- _ -> false
+ {no_match, Doc, {execution_stats, ExecStats1}}
end.
-update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
- Id = couch_util:get_value(id, Props),
- Key = couch_util:get_value(key, Props),
- Cursor#cursor {
+update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props})
+ when Limit > 0 ->
+ Id = couch_util:get_value(<<"_id">>, Props),
+ Cursor#cursor {
bookmark_docid = Id,
bookmark_key = Key
};
update_bookmark_keys(Cursor, _Props) ->
Cursor.
-
-
-%%%%%%%% module tests below %%%%%%%%
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-runs_match_on_doc_with_no_value_test() ->
- Cursor = #cursor {
- db = <<"db">>,
- opts = [],
- execution_stats = #execution_stats{},
- selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]})
- },
- RowProps = [
- {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {doc,{
- [
- {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>},
- {<<"user_id">>,11}
- ]
- }}
- ],
- {Match, _, _} = doc_member(Cursor, RowProps),
- ?assertEqual(Match, no_match).
-
-does_not_run_match_on_doc_with_value_test() ->
- Cursor = #cursor {
- db = <<"db">>,
- opts = [],
- execution_stats = #execution_stats{},
- selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]})
- },
- RowProps = [
- {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {value,1},
- {doc,{
- [
- {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>},
- {<<"user_id">>,11}
- ]
- }}
- ],
- {Match, _, _} = doc_member(Cursor, RowProps),
- ?assertEqual(Match, ok).
-
-
--endif.
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd7..a3572a1 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -18,7 +18,6 @@
incr_keys_examined/1,
incr_docs_examined/1,
incr_docs_examined/2,
- incr_quorum_docs_examined/1,
incr_results_returned/1,
log_start/1,
log_end/1,
@@ -33,7 +32,6 @@ to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
- {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
]}.
@@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) ->
}.
-incr_quorum_docs_examined(Stats) ->
- Stats#execution_stats {
- totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
- }.
-
-
incr_results_returned(Stats) ->
Stats#execution_stats {
resultsReturned = Stats#execution_stats.resultsReturned + 1
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
index ea5ed5e..783c1e7 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -13,7 +13,6 @@
-record(execution_stats, {
totalKeysExamined = 0,
totalDocsExamined = 0,
- totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
executionTimeMs
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
new file mode 100644
index 0000000..99660ac
--- /dev/null
+++ b/src/mango/src/mango_fdb.erl
@@ -0,0 +1,184 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_fdb).
+
+
+-include_lib("fabric/include/fabric2.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+-include("mango_cursor.hrl").
+
+
+-export([
+ create_build_vs/2,
+ set_build_vs/4,
+ get_build_vs/2,
+ get_build_state/2,
+ get_update_seq/2,
+ set_update_seq/3,
+ remove_doc/3,
+ write_doc/3,
+ query/4,
+ base_fold_opts/1,
+ mango_idx_prefix/2
+]).
+
+
+create_build_vs(TxDb, #idx{} = Idx) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = build_vs_key(TxDb, Idx#idx.ddoc),
+ VS = fabric2_fdb:new_versionstamp(Tx),
+ Value = erlfdb_tuple:pack_vs({VS, ?MANGO_INDEX_BUILDING}),
+ ok = erlfdb:set_versionstamped_value(Tx, Key, Value).
+
+
+set_build_vs(TxDb, #idx{} = Idx, VS, State) ->
+ #{
+ tx := Tx
+ } = TxDb,
+
+ Key = build_vs_key(TxDb, Idx#idx.ddoc),
+ Value = erlfdb_tuple:pack({VS, State}),
+ ok = erlfdb:set(Tx, Key, Value).
+
+
+get_build_vs(TxDb, #idx{} = Idx) ->
+ get_build_vs(TxDb, Idx#idx.ddoc);
+
+get_build_vs(TxDb, DDoc) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = build_vs_key(TxDb, DDoc),
+ EV = erlfdb:wait(erlfdb:get(Tx, Key)),
+ if EV == not_found -> not_found; true ->
+ erlfdb_tuple:unpack(EV)
+ end.
+
+
+get_build_state(TxDb, DDoc) ->
+ case get_build_vs(TxDb, DDoc) of
+ not_found -> ?MANGO_INDEX_BUILDING;
+ {_, BuildState} -> BuildState
+ end.
+
+
+get_update_seq(TxDb, #idx{ddoc = DDoc}) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+
+ case erlfdb:wait(erlfdb:get(Tx, seq_key(DbPrefix, DDoc))) of
+ not_found -> <<>>;
+ UpdateSeq -> UpdateSeq
+ end.
+
+
+set_update_seq(TxDb, #idx{ddoc = DDoc}, Seq) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ ok = erlfdb:set(Tx, seq_key(DbPrefix, DDoc), Seq).
+
+
+remove_doc(TxDb, DocId, IdxResults) ->
+ lists:foreach(fun (IdxResult) ->
+ #{
+ ddoc_id := DDocId,
+ results := Results
+ } = IdxResult,
+ MangoIdxPrefix = mango_idx_prefix(TxDb, DDocId),
+ clear_key(TxDb, MangoIdxPrefix, Results, DocId)
+ end, IdxResults).
+
+
+write_doc(TxDb, DocId, IdxResults) ->
+ lists:foreach(fun (IdxResult) ->
+ #{
+ ddoc_id := DDocId,
+ results := Results
+ } = IdxResult,
+ MangoIdxPrefix = mango_idx_prefix(TxDb, DDocId),
+ add_key(TxDb, MangoIdxPrefix, Results, DocId)
+ end, IdxResults).
+
+
+query(Db, CallBack, Cursor, Args) ->
+ #cursor{
+ index = Idx
+ } = Cursor,
+ Mod = mango_idx:fdb_mod(Idx),
+ Mod:query(Db, CallBack, Cursor, Args).
+
+
+base_fold_opts(Args) ->
+ #{
+ dir := Direction,
+ skip := Skip
+ } = Args,
+
+ [
+ {skip, Skip},
+ {dir, Direction},
+ {streaming_mode, want_all},
+ {restart_tx, true}
+ ].
+
+
+mango_idx_prefix(TxDb, Id) ->
+ #{
+ db_prefix := DbPrefix
+ } = TxDb,
+ Key = {?DB_MANGO, Id, ?MANGO_IDX_RANGE},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+seq_key(DbPrefix, DDoc) ->
+ Key = {?DB_MANGO, DDoc, ?MANGO_UPDATE_SEQ},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+build_vs_key(Db, DDoc) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_MANGO, DDoc, ?MANGO_IDX_BUILD_STATUS},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+create_key(MangoIdxPrefix, Results, DocId) ->
+ EncodedResults = couch_views_encoding:encode(Results, key),
+ erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix).
+
+
+clear_key(TxDb, MangoIdxPrefix, Results, DocId) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = create_key(MangoIdxPrefix, Results, DocId),
+ erlfdb:clear(Tx, Key).
+
+
+add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = create_key(MangoIdxPrefix, Results, DocId),
+ Val = couch_views_encoding:encode(Results),
+ erlfdb:set(Tx, Key, Val).
+
diff --git a/src/mango/src/mango_fdb_special.erl b/src/mango/src/mango_fdb_special.erl
new file mode 100644
index 0000000..ba32080
--- /dev/null
+++ b/src/mango/src/mango_fdb_special.erl
@@ -0,0 +1,86 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_fdb_special).
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_cursor.hrl").
+
+
+-export([
+ query/4
+]).
+
+
+query(Db, CallBack, Cursor, Args) ->
+ Acc = #{
+ cursor => Cursor,
+ callback => CallBack
+ },
+ Opts = args_to_fdb_opts(Args),
+ {ok, Acc1} = fabric2_db:fold_docs(Db, fun fold_cb/2, Acc, Opts),
+ {ok, maps:get(cursor, Acc1)}.
+
+
+args_to_fdb_opts(Args) ->
+ #{
+ start_key := StartKey,
+ end_key := EndKey
+ } = Args,
+
+ mango_fdb:base_fold_opts(Args)
+ ++ [{include_docs, true}]
+ ++ start_key_opts(StartKey)
+ ++ end_key_opts(EndKey).
+
+
+start_key_opts(StartKey) ->
+ [{start_key, fabric2_util:encode_all_doc_key(StartKey)}].
+
+
+end_key_opts(?MAX_STR) ->
+ [];
+
+end_key_opts(EndKey) ->
+ [{end_key, fabric2_util:encode_all_doc_key(EndKey)}].
+
+
+fold_cb({row, Props}, Acc) ->
+ #{
+ cursor := Cursor,
+ callback := Callback
+ } = Acc,
+ case is_design_doc(Props) of
+ true ->
+ {ok, Acc};
+ false ->
+ Doc = couch_util:get_value(doc, Props),
+ Key = couch_util:get_value(key, Props),
+ {Go, Cursor1} = Callback({doc, Key, Doc}, Cursor),
+ {Go, Acc#{cursor := Cursor1}}
+ end;
+
+fold_cb(Message, Acc) ->
+ #{
+ cursor := Cursor,
+ callback := Callback
+ } = Acc,
+ {Go, Cursor1} = Callback(Message, Cursor),
+ {Go, Acc#{cursor := Cursor1}}.
+
+
+is_design_doc(RowProps) ->
+ case couch_util:get_value(id, RowProps) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end.
diff --git a/src/mango/src/mango_fdb_view.erl b/src/mango/src/mango_fdb_view.erl
new file mode 100644
index 0000000..bb534e8
--- /dev/null
+++ b/src/mango/src/mango_fdb_view.erl
@@ -0,0 +1,101 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_fdb_view).
+
+
+-export([
+ query/4
+]).
+
+
+-include("mango_idx.hrl").
+-include("mango_cursor.hrl").
+
+
+query(Db, CallBack, Cursor, Args) ->
+ #cursor{
+ index = Idx
+ } = Cursor,
+ MangoIdxPrefix = mango_fdb:mango_idx_prefix(Db, Idx#idx.ddoc),
+ fabric2_fdb:transactional(Db, fun (TxDb) ->
+ Acc0 = #{
+ cursor => Cursor,
+ prefix => MangoIdxPrefix,
+ db => TxDb,
+ callback => CallBack
+ },
+
+ Opts = args_to_fdb_opts(Args),
+ try
+ Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix,
+ fun fold_cb/2, Acc0, Opts),
+ #{
+ cursor := Cursor1
+ } = Acc1,
+ {ok, Cursor1}
+ catch
+ throw:{stop, StopCursor} ->
+ {ok, StopCursor}
+ end
+ end).
+
+
+args_to_fdb_opts(Args) ->
+ #{
+ start_key := StartKey,
+ start_key_docid := StartKeyDocId,
+ end_key := EndKey,
+ end_key_docid := EndKeyDocId
+ } = Args,
+
+ mango_fdb:base_fold_opts(Args)
+ ++ start_key_opts(StartKey, StartKeyDocId)
+ ++ end_key_opts(EndKey, EndKeyDocId).
+
+
+start_key_opts([], _StartKeyDocId) ->
+ [];
+
+start_key_opts(StartKey, StartKeyDocId) ->
+ StartKey1 = couch_views_encoding:encode(StartKey, key),
+ [{start_key, {StartKey1, StartKeyDocId}}].
+
+
+end_key_opts([], _EndKeyDocId) ->
+ [];
+
+end_key_opts(EndKey, EndKeyDocId) ->
+ EndKey1 = couch_views_encoding:encode(EndKey, key),
+ [{end_key, {EndKey1, EndKeyDocId}}].
+
+fold_cb({Key, Val}, Acc) ->
+ #{
+ prefix := MangoIdxPrefix,
+ db := Db,
+ callback := Callback,
+ cursor := Cursor
+
+ } = Acc,
+ {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+ SortKeys = couch_views_encoding:decode(Val),
+ {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]),
+ JSONDoc = couch_doc:to_json_obj(Doc, []),
+ case Callback({doc, SortKeys, JSONDoc}, Cursor) of
+ {ok, Cursor1} ->
+ Acc#{
+ cursor := Cursor1
+ };
+ {stop, Cursor1} ->
+ throw({stop, Cursor1})
+ end.
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 379d2e1..d5e9cfa 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,10 +32,11 @@
threshold = 1490
}).
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
try
- Db = set_user_ctx(Req, Db0),
- handle_req_int(Req, Db)
+ fabric2_fdb:transactional(Db, fun (TxDb) ->
+ handle_req_int(Req, TxDb)
+ end)
catch
throw:{mango_error, Module, Reason} ->
case mango_error:info(Module, Reason) of
@@ -198,11 +199,6 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
- {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
- NewDb.
-
-
get_idx_w_opts(Opts) ->
case lists:keyfind(w, 1, Opts) of
{w, N} when is_integer(N), N > 0 ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5d06a8f..401707b 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -26,6 +26,7 @@
add/2,
remove/2,
from_ddoc/2,
+%% add_build_status/2,
special/1,
dbname/1,
@@ -40,6 +41,7 @@
start_key/2,
end_key/2,
cursor_mod/1,
+ fdb_mod/1,
idx_mod/1,
to_json/1,
delete/4,
@@ -52,10 +54,54 @@
-include("mango.hrl").
-include("mango_idx.hrl").
-
list(Db) ->
- {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
- Indexes.
+ Acc0 = #{
+ db => Db,
+ rows => []
+ },
+ {ok, Indexes} = fabric2_db:fold_design_docs(Db, fun ddoc_fold_cb/2, Acc0, []),
+ Indexes ++ special(Db).
+
+
+ddoc_fold_cb({meta, _}, Acc) ->
+ {ok, Acc};
+
+ddoc_fold_cb(complete, Acc) ->
+ #{rows := Rows} = Acc,
+ {ok, Rows};
+
+ddoc_fold_cb({row, Row}, Acc) ->
+ #{
+ db := Db,
+ rows := Rows
+ } = Acc,
+
+ {Props} = JSONDoc = get_doc(Db, Row),
+
+ case proplists:get_value(<<"language">>, Props) of
+ <<"query">> ->
+ Idx = from_ddoc(Db, JSONDoc),
+ Idx1 = add_build_status(Db, Idx),
+ {ok, Acc#{rows:= Rows ++ Idx1}};
+ _ ->
+ {ok, Acc}
+ end.
+
+
+get_doc(Db, Row) ->
+ {_, Id} = lists:keyfind(id, 1, Row),
+ RevInfo = get_rev_info(Row),
+ Doc = fabric2_fdb:get_doc_body(Db, Id, RevInfo),
+ couch_doc:to_json_obj(Doc, []).
+
+
+get_rev_info(Row) ->
+ {value, {[{rev, RevBin}]}} = lists:keyfind(value, 1, Row),
+ Rev = couch_doc:parse_rev(RevBin),
+ #{
+ rev_id => Rev,
+ rev_path => []
+ }.
get_usable_indexes(Db, Selector, Opts) ->
@@ -63,8 +109,9 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ GlobalIndexes1 = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(GlobalIndexes1 ++ UserSpecifiedIndex),
UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts),
SortFields = get_sort_fields(Opts),
@@ -78,15 +125,17 @@ get_usable_indexes(Db, Selector, Opts) ->
end.
-mango_sort_error(Db, Opts) ->
- case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of
- {false, _} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index});
- {true, true} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned});
- {true, false} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_global})
- end.
+mango_sort_error(_Db, _Opts) ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index}).
+% TODO: add back in when partitions supported
+%% case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of
+%% {false, _} ->
+%% ?MANGO_ERROR({no_usable_index, missing_sort_index});
+%% {true, true} ->
+%% ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned});
+%% {true, false} ->
+%% ?MANGO_ERROR({no_usable_index, missing_sort_index_global})
+%% end.
recover(Db) ->
@@ -182,7 +231,7 @@ from_ddoc(Db, {Props}) ->
_ ->
?MANGO_ERROR(invalid_query_ddoc_language)
end,
- IdxMods = case clouseau_rpc:connected() of
+ IdxMods = case is_text_service_available() of
true ->
[mango_idx_view, mango_idx_text];
false ->
@@ -198,13 +247,26 @@ from_ddoc(Db, {Props}) ->
end, Idxs).
+add_build_status(TxDb, Idxs) ->
+ lists:map(fun
+ (#idx{type = <<"special">>} = Idx) ->
+ Idx;
+ (Idx) ->
+ DDoc = mango_idx:ddoc(Idx),
+ Idx#idx{
+ build_status = mango_fdb:get_build_state(TxDb, DDoc)
+ }
+ end, Idxs).
+
+
special(Db) ->
AllDocs = #idx{
dbname = db_to_name(Db),
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?MANGO_INDEX_READY
},
% Add one for _update_seq
[AllDocs].
@@ -275,6 +337,11 @@ cursor_mod(#idx{type = <<"text">>}) ->
?MANGO_ERROR({index_service_unavailable, <<"text">>})
end.
+fdb_mod(#idx{type = <<"json">>}) ->
+ mango_fdb_view;
+fdb_mod(#idx{def = all_docs, type= <<"special">>}) ->
+ mango_fdb_special.
+
idx_mod(#idx{type = <<"json">>}) ->
mango_idx_view;
@@ -294,7 +361,7 @@ db_to_name(Name) when is_binary(Name) ->
db_to_name(Name) when is_list(Name) ->
iolist_to_binary(Name);
db_to_name(Db) ->
- couch_db:name(Db).
+ maps:get(name, Db).
get_idx_def(Opts) ->
@@ -309,7 +376,7 @@ get_idx_def(Opts) ->
get_idx_type(Opts) ->
case proplists:get_value(type, Opts) of
<<"json">> -> <<"json">>;
- <<"text">> -> case clouseau_rpc:connected() of
+ <<"text">> -> case is_text_service_available() of
true ->
<<"text">>;
false ->
@@ -322,6 +389,11 @@ get_idx_type(Opts) ->
end.
+is_text_service_available() ->
+ erlang:function_exported(clouseau_rpc, connected, 0) andalso
+ clouseau_rpc:connected().
+
+
get_idx_ddoc(Idx, Opts) ->
case proplists:get_value(ddoc, Opts) of
<<"_design/", _Rest/binary>> = Name ->
@@ -407,8 +479,10 @@ set_ddoc_partitioned_option(DDoc, Partitioned) ->
DDoc#doc{body = {NewProps}}.
-get_idx_partitioned(Db, DDocProps) ->
- Default = fabric_util:is_partitioned(Db),
+get_idx_partitioned(_Db, DDocProps) ->
+ % TODO: Add in partition support
+%% Default = fabric_util:is_partitioned(Db),
+ Default = false,
case couch_util:get_value(<<"options">>, DDocProps) of
{DesignOpts} ->
case couch_util:get_value(<<"partitioned">>, DesignOpts) of
@@ -459,7 +533,8 @@ filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
-get_partial_filter_selector(#idx{def = Def}) when Def =:= all_docs; Def =:= undefined ->
+get_partial_filter_selector(#idx{def = Def})
+ when Def =:= all_docs; Def =:= undefined ->
undefined;
get_partial_filter_selector(#idx{def = {Def}}) ->
case proplists:get_value(<<"partial_filter_selector">>, Def) of
@@ -482,14 +557,18 @@ get_legacy_selector(Def) ->
-include_lib("eunit/include/eunit.hrl").
index(SelectorName, Selector) ->
- {
- idx,<<"mango_test_46418cd02081470d93290dc12306ebcb">>,
- <<"_design/57e860dee471f40a2c74ea5b72997b81dda36a24">>,
- <<"Selected">>,<<"json">>,
- {[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
- {SelectorName,{Selector}}]},
- false,
- [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
+ #idx{
+ dbname = <<"mango_test_46418cd02081470d93290dc12306ebcb">>,
+ ddoc = <<"_design/57e860dee471f40a2c74ea5b72997b81dda36a24">>,
+ name = <<"Selected">>,
+ type = <<"json">>,
+ def = {[
+ {<<"fields">>, {[{<<"location">>,<<"asc">>}]}},
+ {SelectorName, {Selector}}
+ ]},
+ partitioned = false,
+ opts = [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}],
+ build_status = undefined
}.
get_partial_filter_all_docs_test() ->
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 9725950..f5f827b 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -17,5 +17,6 @@
type,
def,
partitioned,
- opts
+ opts,
+ build_status
}).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..844a0ba 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -27,6 +27,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
-include("mango_idx.hrl").
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?MANGO_INDEX_READY}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 3791149..949c69b 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -105,7 +105,8 @@ to_json(Idx) ->
{name, Idx#idx.name},
{type, Idx#idx.type},
{partitioned, Idx#idx.partitioned},
- {def, {def_to_json(Idx#idx.def)}}
+ {def, {def_to_json(Idx#idx.def)}},
+ {build_status, Idx#idx.build_status}
]}.
@@ -172,7 +173,7 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [?MAX_JSON_OBJ];
+ [];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
index 0d213e5..2dc3c01 100644
--- a/src/mango/src/mango_idx_view.hrl
+++ b/src/mango/src/mango_idx_view.hrl
@@ -10,4 +10,4 @@
% License for the specific language governing permissions and limitations under
% the License.
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
+-define(MAX_JSON_OBJ, {[{<<"\ufff0">>, <<"\ufff0">>}]}).
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
new file mode 100644
index 0000000..1b4bd40
--- /dev/null
+++ b/src/mango/src/mango_indexer.erl
@@ -0,0 +1,172 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_indexer).
+
+
+-export([
+ create_doc/2,
+ update_doc/3,
+ delete_doc/2
+]).
+
+
+-export([
+ write_doc/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+
+
+create_doc(Db, Doc) ->
+ modify(Db, create, Doc, undefined).
+
+
+update_doc(Db, Doc, PrevDoc) ->
+ modify(Db, update, Doc, PrevDoc).
+
+
+delete_doc(Db, PrevDoc) ->
+ modify(Db, delete, undefined, PrevDoc).
+
+
+modify(Db, Change, Doc, PrevDoc) ->
+ try
+ modify_int(Db, Change, Doc, PrevDoc)
+ catch
+ Error:Reason ->
+ #{
+ name := DbName
+ } = Db,
+
+ Id = doc_id(Doc, PrevDoc),
+ couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+ [DbName, Id, Error, Reason])
+ end,
+ ok.
+
+
+doc_id(undefined, #doc{id = DocId}) ->
+ DocId;
+doc_id(undefined, _) ->
+ <<"unknown_doc_id">>;
+doc_id(#doc{id = DocId}, _) ->
+ DocId.
+
+
+% Check if design doc is mango index and kick off background worker
+% to build the new index
+modify_int(Db, _Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc,
+ _PrevDoc) ->
+ {Props} = JsonDoc = couch_doc:to_json_obj(Doc, []),
+ case proplists:get_value(<<"language">>, Props) of
+ <<"query">> ->
+ [Idx] = mango_idx:from_ddoc(Db, JsonDoc),
+ {ok, _} = mango_jobs:build_index(Db, Idx);
+ _ ->
+ ok
+ end;
+
+modify_int(Db, delete, _, PrevDoc) ->
+ remove_doc(Db, PrevDoc, json_indexes(Db));
+
+modify_int(Db, update, Doc, PrevDoc) ->
+ Indexes = json_indexes(Db),
+ remove_doc(Db, PrevDoc, Indexes),
+ write_doc(Db, Doc, Indexes);
+
+modify_int(Db, create, Doc, _) ->
+ write_doc(Db, Doc, json_indexes(Db)).
+
+
+remove_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ JsonDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ Results = index_doc(Indexes, JsonDoc),
+ mango_fdb:remove_doc(Db, DocId, Results).
+
+
+write_doc(Db, #doc{} = Doc, Indexes) ->
+ #doc{id = DocId} = Doc,
+ JsonDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+ Results = index_doc(Indexes, JsonDoc),
+ mango_fdb:write_doc(Db, DocId, Results).
+
+
+json_indexes(Db) ->
+ lists:filter(fun (Idx) ->
+ Idx#idx.type == <<"json">>
+ end, mango_idx:list(Db)).
+
+
+index_doc(Indexes, Doc) ->
+ lists:foldl(fun(Idx, Acc) ->
+ {IdxDef} = mango_idx:def(Idx),
+ Results = get_index_entries(IdxDef, Doc),
+ case lists:member(not_found, Results) of
+ true ->
+ Acc;
+ false ->
+ IdxResult = #{
+ name => mango_idx:name(Idx),
+ ddoc_id => mango_idx:ddoc(Idx),
+ results => Results
+ },
+ [IdxResult | Acc]
+ end
+ end, [], Indexes).
+
+
+get_index_entries(IdxDef, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxDef),
+ Selector = get_index_partial_filter_selector(IdxDef),
+ case should_index(Selector, Doc) of
+ false ->
+ [not_found];
+ true ->
+ get_index_values(Fields, Doc)
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxDef) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxDef, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the
+ % partial_filter_selector set as selector
+ couch_util:get_value(<<"selector">>, IdxDef, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
diff --git a/src/mango/src/mango_jobs.erl b/src/mango/src/mango_jobs.erl
new file mode 100644
index 0000000..3197f14
--- /dev/null
+++ b/src/mango/src/mango_jobs.erl
@@ -0,0 +1,50 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+
+
+-module(mango_jobs).
+
+-include("mango_idx.hrl").
+-include("mango.hrl").
+
+
+-export([
+ set_timeout/0,
+ build_index/2
+]).
+
+
+set_timeout() ->
+ couch_jobs:set_type_timeout(?MANGO_INDEX_JOB_TYPE, 6).
+
+
+build_index(TxDb, #idx{} = Idx) ->
+ mango_fdb:create_build_vs(TxDb, Idx),
+
+ JobId = job_id(TxDb, Idx),
+ JobData = job_data(TxDb, Idx),
+ ok = couch_jobs:add(TxDb, ?MANGO_INDEX_JOB_TYPE, JobId, JobData),
+ {ok, JobId}.
+
+
+job_id(#{name := DbName}, #idx{ddoc = DDoc} = Idx) ->
+ Cols = iolist_to_binary(mango_idx:columns(Idx)),
+ <<DbName/binary, "_",DDoc/binary, Cols/binary>>.
+
+
+job_data(Db, Idx) ->
+ #{
+ db_name => fabric2_db:name(Db),
+ ddoc_id => mango_idx:ddoc(Idx),
+ columns => mango_idx:columns(Idx),
+ retries => 0
+ }.
+
diff --git a/src/mango/src/mango_jobs_indexer.erl b/src/mango/src/mango_jobs_indexer.erl
new file mode 100644
index 0000000..6cc4547
--- /dev/null
+++ b/src/mango/src/mango_jobs_indexer.erl
@@ -0,0 +1,325 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+% Todo: there is a fair amount of copy-pasta from couch_views_indexer
+% We need to make the indexing generic and have only the specific mango
+% logic here
+-module(mango_jobs_indexer).
+
+
+-export([
+ spawn_link/0
+]).
+
+
+-export([
+ set_timeout/0,
+ init/0
+]).
+
+
+-include("mango.hrl").
+-include("mango_idx.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+
+
+spawn_link() ->
+ proc_lib:spawn_link(?MODULE, init, []).
+
+
+set_timeout() ->
+ mango_jobs:set_timeout().
+
+
+init() ->
+ {ok, Job, Data} = couch_jobs:accept(?MANGO_INDEX_JOB_TYPE, #{}),
+ #{
+ <<"db_name">> := DbName,
+ <<"ddoc_id">> := DDocId,
+ <<"columns">> := JobColumns,
+ <<"retries">> := Retries
+ } = Data,
+
+ {ok, Db} = try
+ fabric2_db:open(DbName, [?ADMIN_CTX])
+ catch error:database_does_not_exist ->
+ couch_jobs:finish(undefined, Job, Data#{
+ error => db_deleted,
+ reason => "Database was deleted"
+ }),
+ exit(normal)
+ end,
+
+ [Idx] = case fabric2_db:open_doc(Db, DDocId) of
+ {ok, DDoc} ->
+ JsonDDoc = couch_doc:to_json_obj(DDoc, []),
+ mango_idx:from_ddoc(Db, JsonDDoc);
+ {not_found, _} ->
+ couch_jobs:finish(undefined, Job, Data#{
+ error => ddoc_deleted,
+ reason => "Mango Index was deleted"
+ }),
+ exit(normal)
+ end,
+
+ Columns = mango_idx:columns(Idx),
+
+ if JobColumns == Columns -> ok; true ->
+ couch_jobs:finish(undefined, Job, Data#{
+ error => index_changed,
+ reason => <<"Mango Index was modified">>
+ }),
+ exit(normal)
+ end,
+
+
+ State = #{
+ tx_db => undefined,
+ idx_vs => undefined,
+ idx_seq => undefined,
+ last_seq => undefined,
+ job => Job,
+ job_data => Data,
+ count => 0,
+ limit => num_changes(),
+ doc_acc => []
+ },
+
+ try
+ update(Db, Idx, State)
+ catch
+ exit:normal ->
+ ok;
+ Error:Reason ->
+ NewRetry = Retries + 1,
+ RetryLimit = retry_limit(),
+
+ case should_retry(NewRetry, RetryLimit, Reason) of
+ true ->
+ DataErr = Data#{<<"retries">> := NewRetry},
+ StateErr = State#{job_data := DataErr},
+ report_progress(StateErr, update);
+ false ->
+ NewData = add_error(Error, Reason, Data),
+ couch_jobs:finish(undefined, Job, NewData),
+ exit(normal)
+ end
+ end.
+
+
+% Transaction limit exceeded don't retry
+should_retry(_, _, {erlfdb_error, 2101}) ->
+ false;
+
+should_retry(Retries, RetryLimit, _) when Retries < RetryLimit ->
+ true;
+
+should_retry(_, _, _) ->
+ false.
+
+
+add_error(error, {erlfdb_error, Code}, Data) ->
+ CodeBin = couch_util:to_binary(Code),
+ CodeString = erlfdb:get_error_string(Code),
+ Data#{
+ error => foundationdb_error,
+ reason => list_to_binary([CodeBin, <<"-">>, CodeString])
+ };
+
+add_error(Error, Reason, Data) ->
+ Data#{
+ error => couch_util:to_binary(Error),
+ reason => couch_util:to_binary(Reason)
+ }.
+
+
+update(#{} = Db, #idx{} = Idx, State0) ->
+ {Idx2, State4} = fabric2_fdb:transactional(Db, fun(TxDb) ->
+
+ State1 = get_update_start_state(TxDb, Idx, State0),
+ {ok, State2} = fold_changes(State1),
+
+ #{
+ idx_vs := IdxVS,
+ count := Count,
+ limit := Limit,
+ doc_acc := DocAcc,
+ idx_seq := IdxSeq
+ } = State2,
+
+ DocAcc1 = couch_views_indexer:fetch_docs(TxDb, DocAcc),
+ index_docs(TxDb, Idx, DocAcc1),
+ mango_fdb:set_update_seq(TxDb, Idx, IdxSeq),
+ case Count < Limit of
+ true ->
+ mango_fdb:set_build_vs(TxDb, Idx, IdxVS, ?MANGO_INDEX_READY),
+ report_progress(State2, finished),
+ {Idx, finished};
+ false ->
+ State3 = report_progress(State2, update),
+ {Idx, State3#{
+ tx_db := undefined,
+ count := 0,
+ doc_acc := [],
+ idx_seq := IdxSeq
+ }}
+ end
+ end),
+
+ case State4 of
+ finished ->
+ ok;
+ _ ->
+ update(Db, Idx2, State4)
+ end.
+
+
+get_update_start_state(TxDb, Idx, #{idx_vs := undefined} = State) ->
+ #{
+ job := Job,
+ job_data := Data
+ } = State,
+
+ {IdxVS, BuildState} = mango_fdb:get_build_vs(TxDb, Idx),
+ if BuildState == ?MANGO_INDEX_BUILDING -> ok; true ->
+ couch_jobs:finish(undefined, Job, Data#{
+ error => index_built,
+ reason => <<"Index is already built">>
+ }),
+ exit(normal)
+ end,
+
+ IdxSeq = mango_fdb:get_update_seq(TxDb, Idx),
+
+ State#{
+ tx_db := TxDb,
+ idx_vs := IdxVS,
+ idx_seq := IdxSeq
+ };
+
+get_update_start_state(TxDb, _Idx, State) ->
+ State#{
+ tx_db := TxDb
+ }.
+
+
+fold_changes(State) ->
+ #{
+ idx_seq := SinceSeq,
+ limit := Limit,
+ tx_db := TxDb
+ } = State,
+
+ Fun = fun process_changes/2,
+ Opts = [{limit, Limit}, {restart_tx, false}],
+ fabric2_db:fold_changes(TxDb, SinceSeq, Fun, State, Opts).
+
+
+process_changes(Change, Acc) ->
+ #{
+ doc_acc := DocAcc,
+ count := Count,
+ idx_vs := IdxVS
+ } = Acc,
+
+ #{
+ id := Id,
+ sequence := LastSeq
+ } = Change,
+
+ DocVS = fabric2_fdb:next_vs(fabric2_fdb:seq_to_vs(LastSeq)),
+
+ case IdxVS =< DocVS of
+ true ->
+ {stop, Acc};
+ false ->
+ Acc1 = case Id of
+ <<?DESIGN_DOC_PREFIX, _/binary>> ->
+ maps:merge(Acc, #{
+ count => Count + 1,
+ idx_seq => LastSeq
+ });
+ _ ->
+ Acc#{
+ doc_acc := DocAcc ++ [Change],
+ count := Count + 1,
+ idx_seq := LastSeq
+ }
+ end,
+ {ok, Acc1}
+ end.
+
+
+index_docs(Db, Idx, Docs) ->
+ lists:foreach(fun (Doc) ->
+ index_doc(Db, Idx, Doc)
+ end, Docs).
+
+
+index_doc(_Db, _Idx, #{deleted := true}) ->
+ ok;
+
+index_doc(Db, Idx, #{doc := Doc}) ->
+ mango_indexer:write_doc(Db, Doc, [Idx]).
+
+
+report_progress(State, UpdateType) ->
+ #{
+ tx_db := TxDb,
+ job := Job1,
+ job_data := JobData
+ } = State,
+
+ #{
+ <<"db_name">> := DbName,
+ <<"ddoc_id">> := DDocId,
+ <<"columns">> := Columns,
+ <<"retries">> := Retries
+ } = JobData,
+
+ % Reconstruct from scratch to remove any
+ % possible existing error state.
+ NewData = #{
+ <<"db_name">> => DbName,
+ <<"ddoc_id">> => DDocId,
+ <<"columns">> => Columns,
+ <<"retries">> => Retries
+ },
+
+ case UpdateType of
+ update ->
+ case couch_jobs:update(TxDb, Job1, NewData) of
+ {ok, Job2} ->
+ State#{job := Job2};
+ {error, halt} ->
+ couch_log:error("~s job halted :: ~w", [?MODULE, Job1]),
+ exit(normal)
+ end;
+ finished ->
+ case couch_jobs:finish(TxDb, Job1, NewData) of
+ ok ->
+ State;
+ {error, halt} ->
+ couch_log:error("~s job halted :: ~w", [?MODULE, Job1]),
+ exit(normal)
+ end
+ end.
+
+
+num_changes() ->
+ config:get_integer("mango", "change_limit", 100).
+
+
+retry_limit() ->
+ config:get_integer("mango", "retry_limit", 3).
diff --git a/src/mango/src/mango_json_bookmark.erl b/src/mango/src/mango_json_bookmark.erl
index 97f81cf..edc83cf 100644
--- a/src/mango/src/mango_json_bookmark.erl
+++ b/src/mango/src/mango_json_bookmark.erl
@@ -23,23 +23,28 @@
-include("mango_cursor.hrl").
-include("mango.hrl").
-update_args(EncodedBookmark, #mrargs{skip = Skip} = Args) ->
+update_args(EncodedBookmark, FdbOpts) ->
+ #{
+ skip := Skip
+ } = FdbOpts,
Bookmark = unpack(EncodedBookmark),
case is_list(Bookmark) of
true ->
{startkey, Startkey} = lists:keyfind(startkey, 1, Bookmark),
- {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1, Bookmark),
- Args#mrargs{
- start_key = Startkey,
- start_key_docid = StartkeyDocId,
- skip = 1 + Skip
+ {startkey_docid, StartkeyDocId} = lists:keyfind(startkey_docid, 1,
+ Bookmark),
+ FdbOpts#{
+ start_key => Startkey,
+ start_key_docid => StartkeyDocId,
+ skip => 1 + Skip
};
false ->
- Args
+ FdbOpts
end.
-create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey}) when BookmarkKey =/= undefined ->
+create(#cursor{bookmark_docid = BookmarkDocId, bookmark_key = BookmarkKey})
+ when BookmarkKey =/= undefined ->
QueryArgs = [
{startkey_docid, BookmarkDocId},
{startkey, BookmarkKey}
@@ -61,7 +66,8 @@ unpack(Packed) ->
end.
verify(Bookmark) when is_list(Bookmark) ->
- case lists:keymember(startkey, 1, Bookmark) andalso lists:keymember(startkey_docid, 1, Bookmark) of
+ case lists:keymember(startkey, 1, Bookmark)
+ andalso lists:keymember(startkey_docid, 1, Bookmark) of
true -> Bookmark;
_ -> throw(invalid_bookmark)
end;
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
index cbf3622..5a05083 100644
--- a/src/mango/src/mango_native_proc.erl
+++ b/src/mango/src/mango_native_proc.erl
@@ -47,7 +47,8 @@
start_link() ->
- gen_server:start_link(?MODULE, [], []).
+ throw({error, mango_native_proc_is_no_longer_needed}).
+%% gen_server:start_link(?MODULE, [], []).
set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
diff --git a/src/mango/src/mango_sup.erl b/src/mango/src/mango_sup.erl
index b0dedf1..d702d09 100644
--- a/src/mango/src/mango_sup.erl
+++ b/src/mango/src/mango_sup.erl
@@ -21,4 +21,17 @@ start_link(Args) ->
supervisor:start_link({local,?MODULE}, ?MODULE, Args).
init([]) ->
- {ok, {{one_for_one, 3, 10}, couch_epi:register_service(mango_epi, [])}}.
+ Flags = #{
+ strategy => one_for_one,
+ intensity => 3,
+ period => 10
+ },
+
+ Args = [{worker, mango_jobs_indexer}],
+ Children = [
+ #{
+ id => mango_indexer_server,
+ start => {couch_views_server, start_link, [Args]}
+ }
+ ] ++ couch_epi:register_service(mango_epi, []),
+ {ok, {Flags, Children}}.
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index a734717..faad55c 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,7 @@ open_doc(Db, DocId) ->
open_doc(Db, DocId, Options) ->
- case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
- {ok, Doc} ->
- {ok, Doc};
- {not_found, _} ->
- not_found;
- _ ->
- ?MANGO_ERROR({error_loading_doc, DocId})
- end.
+ fabric2_db:open_doc(Db, DocId, Options).
open_ddocs(Db) ->
@@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
case open_doc(Db, DDocId, DbOpts) of
{ok, Doc} ->
{ok, check_lang(Doc)};
- not_found ->
+ {not_found, missing} ->
Body = {[
{<<"language">>, <<"query">>}
]},
diff --git a/src/mango/test/01-index-crud-test.py b/src/mango/test/01-index-crud-test.py
index b602399..e72b216 100644
--- a/src/mango/test/01-index-crud-test.py
+++ b/src/mango/test/01-index-crud-test.py
@@ -113,6 +113,22 @@ class IndexCrudTests(mango.DbPerClass):
return
raise AssertionError("index not created")
+ @unittest.skip("need spidermonkey 60")
+ def test_ignore_design_docs(self):
+ fields = ["baz", "foo"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+ self.db.save_doc({
+ "_id": "_design/ignore",
+ "views": {
+ "view1": {
+ "map": "function (doc) { emit(doc._id, 1)}"
+ }
+ }
+ })
+ Indexes = self.db.list_indexes()
+ self.assertEqual(len(Indexes), 2)
+
def test_read_idx_doc(self):
self.db.create_index(["foo", "bar"], name="idx_01")
self.db.create_index(["hello", "bar"])
@@ -122,8 +138,8 @@ class IndexCrudTests(mango.DbPerClass):
ddocid = idx["ddoc"]
doc = self.db.open_doc(ddocid)
self.assertEqual(doc["_id"], ddocid)
- info = self.db.ddoc_info(ddocid)
- self.assertEqual(info["name"], ddocid.split("_design/")[-1])
+ # info = self.db.ddoc_info(ddocid)
+ # self.assertEqual(info["name"], ddocid.split("_design/")[-1])
def test_delete_idx_escaped(self):
self.db.create_index(["foo", "bar"], name="idx_01")
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 0fc4248..00d1d84 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -287,12 +287,12 @@ class BasicFindTests(mango.UserDocsTests):
def test_explain_view_args(self):
explain = self.db.find({"age": {"$gt": 0}}, fields=["manager"], explain=True)
- assert explain["mrargs"]["stable"] == False
- assert explain["mrargs"]["update"] == True
- assert explain["mrargs"]["reduce"] == False
- assert explain["mrargs"]["start_key"] == [0]
- assert explain["mrargs"]["end_key"] == ["<MAX>"]
- assert explain["mrargs"]["include_docs"] == True
+ assert explain["args"]["stable"] == False
+ assert explain["args"]["update"] == True
+ assert explain["args"]["reduce"] == False
+ assert explain["args"]["start_key"] == [0]
+ assert explain["args"]["end_key"] == ["<MAX>"]
+ assert explain["args"]["include_docs"] == True
def test_sort_with_all_docs(self):
explain = self.db.find(
diff --git a/src/mango/test/03-operator-test.py b/src/mango/test/03-operator-test.py
index 935f470..fdcd079 100644
--- a/src/mango/test/03-operator-test.py
+++ b/src/mango/test/03-operator-test.py
@@ -190,5 +190,5 @@ class OperatorAllDocsTests(mango.UserDocsTestsNoIndexes, OperatorTests):
doc_id = "8e1c90c0-ac18-4832-8081-40d14325bde0"
r = self.db.find({"_id": doc_id}, explain=True, return_raw=True)
- self.assertEqual(r["mrargs"]["end_key"], doc_id)
- self.assertEqual(r["mrargs"]["start_key"], doc_id)
+ self.assertEqual(r["args"]["end_key"], doc_id)
+ self.assertEqual(r["args"]["start_key"], doc_id)
diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py
index 3f7fb9f..e6d74bd 100644
--- a/src/mango/test/05-index-selection-test.py
+++ b/src/mango/test/05-index-selection-test.py
@@ -183,6 +183,7 @@ class IndexSelectionTests:
# This doc will not be saved given the new ddoc validation code
# in couch_mrview
+ @unittest.skip("need to add couch_mrview:validate_ddoc_fields")
def test_manual_bad_view_idx01(self):
design_doc = {
"_id": "_design/bad_view_index",
diff --git a/src/mango/test/11-ignore-design-docs-test.py b/src/mango/test/11-ignore-design-docs-test.py
index f31dcc5..fd9b688 100644
--- a/src/mango/test/11-ignore-design-docs-test.py
+++ b/src/mango/test/11-ignore-design-docs-test.py
@@ -16,7 +16,7 @@ import unittest
DOCS = [
{"_id": "_design/my-design-doc"},
{"_id": "54af50626de419f5109c962f", "user_id": 0, "age": 10, "name": "Jimi"},
- {"_id": "54af50622071121b25402dc3", "user_id": 1, "age": 11, "name": "Eddie"},
+ {"_id": "54af50622071121b25402dc3", "user_id": 1, "age": 11, "name": "Eddie"}
]
diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py
index 2de88a2..d495e94 100644
--- a/src/mango/test/12-use-correct-index-test.py
+++ b/src/mango/test/12-use-correct-index-test.py
@@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs(copy.deepcopy(DOCS))
def test_choose_index_with_one_field_in_index(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie"}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
def test_choose_index_with_two(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_alphabetically(self):
- self.db.create_index(["name"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/aaa")
def test_choose_index_most_accurate(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_most_accurate_in_memory_selector(self):
- self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
@@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
def test_chooses_idxA(self):
DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}]
self.db.save_docs(copy.deepcopy(DOCS2))
- self.db.create_index(["a", "b", "c"])
- self.db.create_index(["a", "d", "e"])
+ self.db.create_index(["a", "b", "c"], wait_for_built_index=False)
+ self.db.create_index(["a", "d", "e"], wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find(
{"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True
)
@@ -117,7 +123,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.assertEqual(len(docs), 1)
explain = self.db.find(selector, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
- self.assertEqual(explain["mrargs"]["end_key"], [10, "<MAX>"])
+ self.assertEqual(explain["args"]["end_key"], [10, "<MAX>"])
# all documents contain an _id and _rev field they
# should not be used to restrict indexes based on the
diff --git a/src/mango/test/13-stable-update-test.py b/src/mango/test/13-stable-update-test.py
index 348ac5e..1af63b9 100644
--- a/src/mango/test/13-stable-update-test.py
+++ b/src/mango/test/13-stable-update-test.py
@@ -12,6 +12,7 @@
import copy
import mango
+import unittest
DOCS1 = [
{
@@ -39,6 +40,7 @@ class SupportStableAndUpdate(mango.DbPerClass):
self.db.create_index(["name"])
self.db.save_docs(copy.deepcopy(DOCS1))
+ @unittest.skip("this FDB doesn't support this")
def test_update_updates_view_when_specified(self):
docs = self.db.find({"name": "Eddie"}, update=False)
assert len(docs) == 0
diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py
index 73d15ea..25f5385 100644
--- a/src/mango/test/13-users-db-find-test.py
+++ b/src/mango/test/13-users-db-find-test.py
@@ -12,63 +12,65 @@
# the License.
-import mango, requests
+import mango, requests, unittest
-class UsersDbFindTests(mango.UsersDbTests):
- def test_simple_find(self):
- docs = self.db.find({"name": {"$eq": "demo02"}})
- assert len(docs) == 1
- assert docs[0]["_id"] == "org.couchdb.user:demo02"
-
- def test_multi_cond_and(self):
- self.db.create_index(["type", "roles"])
- docs = self.db.find({"type": "user", "roles": {"$eq": ["reader"]}})
- assert len(docs) == 1
- assert docs[0]["_id"] == "org.couchdb.user:demo02"
-
- def test_multi_cond_or(self):
- docs = self.db.find(
- {"$and": [{"type": "user"}, {"$or": [{"order": 1}, {"order": 3}]}]}
- )
- assert len(docs) == 2
- assert docs[0]["_id"] == "org.couchdb.user:demo01"
- assert docs[1]["_id"] == "org.couchdb.user:demo03"
-
- def test_sort(self):
- self.db.create_index(["order", "name"])
- selector = {"name": {"$gt": "demo01"}}
- docs1 = self.db.find(selector, sort=[{"order": "asc"}])
- docs2 = list(sorted(docs1, key=lambda d: d["order"]))
- assert docs1 is not docs2 and docs1 == docs2
-
- docs1 = self.db.find(selector, sort=[{"order": "desc"}])
- docs2 = list(reversed(sorted(docs1, key=lambda d: d["order"])))
- assert docs1 is not docs2 and docs1 == docs2
-
- def test_fields(self):
- selector = {"name": {"$eq": "demo02"}}
- docs = self.db.find(selector, fields=["name", "order"])
- assert len(docs) == 1
- assert sorted(docs[0].keys()) == ["name", "order"]
-
- def test_empty(self):
- docs = self.db.find({})
- assert len(docs) == 3
-
-
-class UsersDbIndexFindTests(UsersDbFindTests):
- def setUp(self):
- self.db.create_index(["name"])
-
- def test_multi_cond_and(self):
- self.db.create_index(["type", "roles"])
- super(UsersDbIndexFindTests, self).test_multi_cond_and()
-
- def test_multi_cond_or(self):
- self.db.create_index(["type", "order"])
- super(UsersDbIndexFindTests, self).test_multi_cond_or()
-
- def test_sort(self):
- self.db.create_index(["order", "name"])
- super(UsersDbIndexFindTests, self).test_sort()
+# @unittest.skip("this FDB doesn't support this")
+# class UsersDbFindTests(mango.UsersDbTests):
+# def test_simple_find(self):
+# docs = self.db.find({"name": {"$eq": "demo02"}})
+# assert len(docs) == 1
+# assert docs[0]["_id"] == "org.couchdb.user:demo02"
+#
+# def test_multi_cond_and(self):
+# self.db.create_index(["type", "roles"])
+# docs = self.db.find({"type": "user", "roles": {"$eq": ["reader"]}})
+# assert len(docs) == 1
+# assert docs[0]["_id"] == "org.couchdb.user:demo02"
+#
+# def test_multi_cond_or(self):
+# docs = self.db.find(
+# {"$and": [{"type": "user"}, {"$or": [{"order": 1}, {"order": 3}]}]}
+# )
+# assert len(docs) == 2
+# assert docs[0]["_id"] == "org.couchdb.user:demo01"
+# assert docs[1]["_id"] == "org.couchdb.user:demo03"
+#
+# def test_sort(self):
+# self.db.create_index(["order", "name"])
+# selector = {"name": {"$gt": "demo01"}}
+# docs1 = self.db.find(selector, sort=[{"order": "asc"}])
+# docs2 = list(sorted(docs1, key=lambda d: d["order"]))
+# assert docs1 is not docs2 and docs1 == docs2
+#
+# docs1 = self.db.find(selector, sort=[{"order": "desc"}])
+# docs2 = list(reversed(sorted(docs1, key=lambda d: d["order"])))
+# assert docs1 is not docs2 and docs1 == docs2
+#
+# def test_fields(self):
+# selector = {"name": {"$eq": "demo02"}}
+# docs = self.db.find(selector, fields=["name", "order"])
+# assert len(docs) == 1
+# assert sorted(docs[0].keys()) == ["name", "order"]
+#
+# def test_empty(self):
+# docs = self.db.find({})
+# assert len(docs) == 3
+#
+#
+# @unittest.skip("this FDB doesn't support this")
+# class UsersDbIndexFindTests(UsersDbFindTests):
+# def setUp(self):
+# self.db.create_index(["name"])
+#
+# def test_multi_cond_and(self):
+# self.db.create_index(["type", "roles"])
+# super(UsersDbIndexFindTests, self).test_multi_cond_and()
+#
+# def test_multi_cond_or(self):
+# self.db.create_index(["type", "order"])
+# super(UsersDbIndexFindTests, self).test_multi_cond_or()
+#
+# def test_sort(self):
+# self.db.create_index(["order", "name"])
+# super(UsersDbIndexFindTests, self).test_sort()
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 922cadf..0ac8a3d 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -39,8 +38,7 @@ class ExecutionStatsTests(mango.UserDocsTests):
)
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -63,7 +61,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
self.assertEqual(len(resp["docs"]), 1)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
diff --git a/src/mango/test/16-index-selectors-test.py b/src/mango/test/16-index-selectors-test.py
index 4510065..a3014e9 100644
--- a/src/mango/test/16-index-selectors-test.py
+++ b/src/mango/test/16-index-selectors-test.py
@@ -158,6 +158,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_with_no_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolnoselectorddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschoolnoselector")
self.assertEqual(resp["index"]["name"], "oldschoolnoselector")
docs = self.db.find(selector, use_index="oldschoolnoselector")
@@ -166,6 +167,7 @@ class IndexSelectorJson(mango.DbPerClass):
def test_old_selector_still_supported(self):
selector = {"location": {"$gte": "FRA"}}
self.db.save_doc(oldschoolddoc)
+ self.db.wait_for_built_indexes()
resp = self.db.find(selector, explain=True, use_index="oldschool")
self.assertEqual(resp["index"]["name"], "oldschool")
docs = self.db.find(selector, use_index="oldschool")
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
index 21e7afd..5a8fced 100644
--- a/src/mango/test/17-multi-type-value-test.py
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -53,9 +53,9 @@ class MultiValueFieldTests:
class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["name"], wait_for_built_index=False)
+ self.db.create_index(["age", "name"], wait_for_built_index=True)
self.db.save_docs(copy.deepcopy(DOCS))
- self.db.create_index(["name"])
- self.db.create_index(["age", "name"])
# @unittest.skipUnless(mango.has_text_service(), "requires text service")
diff --git a/src/mango/test/18-json-sort.py b/src/mango/test/18-json-sort.py
index d4e60a3..62c8e29 100644
--- a/src/mango/test/18-json-sort.py
+++ b/src/mango/test/18-json-sort.py
@@ -15,7 +15,7 @@ import copy
import unittest
DOCS = [
- {"_id": "1", "name": "Jimi", "age": 10, "cars": 1},
+ {"_id": "aa", "name": "Jimi", "age": 10, "cars": 1},
{"_id": "2", "name": "Eddie", "age": 20, "cars": 1},
{"_id": "3", "name": "Jane", "age": 30, "cars": 2},
{"_id": "4", "name": "Mary", "age": 40, "cars": 2},
@@ -33,7 +33,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=["age"], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "fwd")
+ self.assertEqual(explain["args"]["direction"], "fwd")
def test_works_for_all_fields_specified(self):
self.db.create_index(["cars", "age"], name="cars-age")
@@ -52,7 +52,7 @@ class JSONIndexSortOptimisations(mango.DbPerClass):
selector = {"cars": "2", "age": {"$gt": 10}}
explain = self.db.find(selector, sort=[{"age": "desc"}], explain=True)
self.assertEqual(explain["index"]["name"], "cars-age")
- self.assertEqual(explain["mrargs"]["direction"], "rev")
+ self.assertEqual(explain["args"]["direction"], "rev")
def test_not_work_for_non_constant_field(self):
self.db.create_index(["cars", "age"], name="cars-age")
diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py
index bf865d6..3673ca7 100644
--- a/src/mango/test/19-find-conflicts.py
+++ b/src/mango/test/19-find-conflicts.py
@@ -12,12 +12,13 @@
import mango
import copy
+import unittest
DOC = [{"_id": "doc", "a": 2}]
CONFLICT = [{"_id": "doc", "_rev": "1-23202479633c2b380f79507a776743d5", "a": 1}]
-
+@unittest.skip("re-enable once conflicts are supported")
class ChooseCorrectIndexForDocs(mango.DbPerClass):
def setUp(self):
self.db.recreate()
diff --git a/src/mango/test/20-no-timeout-test.py b/src/mango/test/20-no-timeout-test.py
index cffdfc3..b54e81c 100644
--- a/src/mango/test/20-no-timeout-test.py
+++ b/src/mango/test/20-no-timeout-test.py
@@ -18,15 +18,22 @@ import unittest
class LongRunningMangoTest(mango.DbPerClass):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["value"])
docs = []
for i in range(100000):
- docs.append({"_id": str(i), "another": "field"})
- if i % 20000 == 0:
+ docs.append({"_id": str(i), "another": "field", "value": i})
+ if i % 1000 == 0:
self.db.save_docs(docs)
docs = []
# This test should run to completion and not timeout
def test_query_does_not_time_out(self):
- selector = {"_id": {"$gt": 0}, "another": "wrong"}
- docs = self.db.find(selector)
+ # using _all_docs
+ selector1 = {"_id": {"$gt": 0}, "another": "wrong"}
+ docs = self.db.find(selector1)
+ self.assertEqual(len(docs), 0)
+
+ # using index
+ selector2 = {"value": {"$gt": 0}, "another": "wrong"}
+ docs = self.db.find(selector2)
self.assertEqual(len(docs), 0)
diff --git a/src/mango/test/eunit/mango_indexer_test.erl b/src/mango/test/eunit/mango_indexer_test.erl
new file mode 100644
index 0000000..80587c4
--- /dev/null
+++ b/src/mango/test/eunit/mango_indexer_test.erl
@@ -0,0 +1,164 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_indexer_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("mango/src/mango_cursor.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [with([
+ ?TDEF(index_docs),
+ ?TDEF(update_doc),
+ ?TDEF(delete_doc)
+ ])]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ mango
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_idx_ddoc(Db),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Docs = make_docs(3),
+ fabric2_db:update_docs(Db, Docs),
+ {Db, couch_doc:to_json_obj(DDoc, [])}.
+
+
+foreach_teardown({Db, _}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+update_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"2">>}, {value, 4}]
+ ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+run_query(Db, DDoc) ->
+ Args = #{
+ start_key => [],
+ start_key_docid => <<>>,
+ end_key => [],
+ end_key_docid => <<255>>,
+ dir => fwd,
+ skip => 0
+ },
+ [Idx] = mango_idx:from_ddoc(Db, DDoc),
+ Cursor = #cursor{
+ db = Db,
+ index = Idx,
+ user_acc = []
+ },
+ {ok, Cursor1} = mango_fdb:query(Db, fun query_cb/2, Cursor, Args),
+ Acc = Cursor1#cursor.user_acc,
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props)}
+ ]
+
+ end, Acc).
+
+
+create_idx_ddoc(Db) ->
+ Opts = [
+ {def, {[{<<"fields">>,{[{<<"value">>,<<"asc">>}]}}]}},
+ {type, <<"json">>},
+ {name, <<"idx_01">>},
+ {ddoc, auto_name},
+ {w, 3},
+ {partitioned, db_default}
+ ],
+
+ {ok, Idx} = mango_idx:new(Db, Opts),
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), []),
+ {ok, NewDDoc} = mango_idx:add(DDoc, Idx),
+ NewDDoc.
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({doc, _, Doc}, #cursor{user_acc = Acc} = Cursor) ->
+ {ok, Cursor#cursor{
+ user_acc = Acc ++ [Doc]
+ }}.
+
diff --git a/src/mango/test/eunit/mango_jobs_indexer_test.erl b/src/mango/test/eunit/mango_jobs_indexer_test.erl
new file mode 100644
index 0000000..928861b
--- /dev/null
+++ b/src/mango/test/eunit/mango_jobs_indexer_test.erl
@@ -0,0 +1,189 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_jobs_indexer_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("mango/src/mango.hrl").
+-include_lib("mango/src/mango_cursor.hrl").
+-include_lib("mango/src/mango_idx.hrl").
+
+-include_lib("fabric/test/fabric2_test.hrl").
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [
+ with([?TDEF(index_docs)]),
+ with([?TDEF(index_lots_of_docs, 10)]),
+ with([?TDEF(index_can_recover_from_crash, 60)])
+ ]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ mango
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]),
+ Db.
+
+
+foreach_teardown(Db) ->
+ meck:unload(),
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs(Db) ->
+ DDoc = generate_docs(Db, 5),
+ wait_while_ddoc_builds(Db),
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"4">>}, {value, 4}],
+ [{id, <<"5">>}, {value, 5}]
+ ], Docs).
+
+
+index_lots_of_docs(Db) ->
+ DDoc = generate_docs(Db, 150),
+ wait_while_ddoc_builds(Db),
+ Docs = run_query(Db, DDoc),
+ ?assertEqual(length(Docs), 150).
+
+
+index_can_recover_from_crash(Db) ->
+ meck:new(mango_indexer, [passthrough]),
+ meck:expect(mango_indexer, write_doc, fun (TxDb, Doc, Idxs) ->
+ Id = Doc#doc.id,
+ case Id == <<"2">> of
+ true ->
+ meck:unload(mango_indexer),
+ throw({fake_crash, test_jobs_restart});
+ false ->
+ meck:passthrough([TxDb, Doc, Idxs])
+ end
+ end),
+ DDoc = generate_docs(Db, 3),
+ wait_while_ddoc_builds(Db),
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+wait_while_ddoc_builds(Db) ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Ready = lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?MANGO_INDEX_READY
+ end, mango_idx:list(TxDb)),
+
+ if length(Ready) > 1 -> ok; true ->
+ timer:sleep(100),
+ wait_while_ddoc_builds(Db)
+ end
+ end).
+
+
+run_query(Db, DDoc) ->
+ Args = #{
+ start_key => [],
+ start_key_docid => <<>>,
+ end_key => [],
+ end_key_docid => <<255>>,
+ dir => fwd,
+ skip => 0
+ },
+ [Idx] = mango_idx:from_ddoc(Db, DDoc),
+ Cursor = #cursor{
+ db = Db,
+ index = Idx,
+ user_acc = []
+ },
+ {ok, Cursor1} = mango_fdb:query(Db, fun query_cb/2, Cursor, Args),
+ Acc = Cursor1#cursor.user_acc,
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props)}
+ ]
+
+ end, Acc).
+
+
+generate_docs(Db, Count) ->
+ Docs = make_docs(Count),
+ fabric2_db:update_docs(Db, Docs),
+
+
+ DDoc = create_idx_ddoc(Db),
+ fabric2_db:update_docs(Db, [DDoc]),
+ couch_doc:to_json_obj(DDoc, []).
+
+
+create_idx_ddoc(Db) ->
+ Opts = [
+ {def, {[{<<"fields">>,{[{<<"value">>,<<"asc">>}]}}]}},
+ {type, <<"json">>},
+ {name, <<"idx_01">>},
+ {ddoc, auto_name},
+ {w, 3},
+ {partitioned, db_default}
+ ],
+
+ {ok, Idx} = mango_idx:new(Db, Opts),
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), []),
+ {ok, NewDDoc} = mango_idx:add(DDoc, Idx),
+ NewDDoc.
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({doc, _, Doc}, #cursor{user_acc = Acc} = Cursor) ->
+ {ok, Cursor#cursor{
+ user_acc = Acc ++ [Doc]
+ }}.
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index de8a638..62d6c1b 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -48,8 +48,8 @@ class Database(object):
dbname,
host="127.0.0.1",
port="15984",
- user="testuser",
- password="testpass",
+ user="adm",
+ password="pass",
):
root_url = get_from_environment("COUCH_HOST", "http://{}:{}".format(host, port))
auth_header = get_from_environment("COUCH_AUTH_HEADER", None)
@@ -139,6 +139,7 @@ class Database(object):
ddoc=None,
partial_filter_selector=None,
selector=None,
+ wait_for_built_index=True
):
body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
if name is not None:
@@ -156,13 +157,27 @@ class Database(object):
assert r.json()["name"] is not None
created = r.json()["result"] == "created"
- if created:
- # wait until the database reports the index as available
- while len(self.get_index(r.json()["id"], r.json()["name"])) < 1:
- delay(t=0.1)
+ if created and wait_for_built_index:
+ # wait until the database reports the index as available and build
+ while len([
+ i
+ for i in self.get_index(r.json()["id"], r.json()["name"])
+ if i["build_status"] == "ready"
+ ]) < 1:
+ delay(t=0.2)
return created
+ def wait_for_built_indexes(self):
+ indexes = self.list_indexes()
+ while len([
+ i
+ for i in self.list_indexes()
+ if i["build_status"] == "ready"
+ ]) < len(indexes):
+ delay(t=0.2)
+
+
def create_text_index(
self,
analyzer=None,
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index e049535..6793601 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -61,11 +61,11 @@ def setup_users(db, **kwargs):
def setup(db, index_type="view", **kwargs):
db.recreate()
- db.save_docs(copy.deepcopy(DOCS))
if index_type == "view":
add_view_indexes(db, kwargs)
elif index_type == "text":
add_text_indexes(db, kwargs)
+ db.save_docs(copy.deepcopy(DOCS))
def add_view_indexes(db, kwargs):
@@ -90,7 +90,9 @@ def add_view_indexes(db, kwargs):
(["ordered"], "ordered"),
]
for (idx, name) in indexes:
- assert db.create_index(idx, name=name, ddoc=name) is True
+ assert db.create_index(idx, name=name, ddoc=name,
+ wait_for_built_index=False) is True
+ db.wait_for_built_indexes()
def add_text_indexes(db, kwargs):