You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/02/19 08:22:35 UTC
[couchdb] 23/23: code clean up
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch fdb-mango-indexes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 506adf0e99624a9fe1ed74605b413ef3e2a2cea2
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Feb 18 14:44:04 2020 +0200
code clean up
---
src/couch_views/src/couch_views_server.erl | 1 -
src/fabric/src/fabric2_fdb.erl | 1 -
src/mango/src/mango_app.erl | 1 +
src/mango/src/mango_cursor_view.erl | 60 +++++-----------------
src/mango/src/mango_fdb.erl | 80 +++---------------------------
src/mango/src/mango_fdb_special.erl | 64 ++++++++++++++++++++++--
src/mango/src/mango_fdb_view.erl | 72 +++++++++++++++++++++++++--
src/mango/src/mango_indexer.erl | 3 +-
8 files changed, 150 insertions(+), 132 deletions(-)
diff --git a/src/couch_views/src/couch_views_server.erl b/src/couch_views/src/couch_views_server.erl
index 6299407..1a8caf3 100644
--- a/src/couch_views/src/couch_views_server.erl
+++ b/src/couch_views/src/couch_views_server.erl
@@ -92,7 +92,6 @@ spawn_workers(St) ->
max_workers := MaxWorkers,
worker_module := WorkerModule
} = St,
- io:format("BOOM COUCH VIEWS SERVER ~p ~n", [WorkerModule]),
case maps:size(Workers) < MaxWorkers of
true ->
Pid = WorkerModule:spawn_link(),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index cf73244..4249701 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -811,7 +811,6 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Update database size
AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
RemSize = sum_rem_rev_sizes(ToRemove),
-%% TODO: causing mango indexes to fail with fdb error 1036
%% incr_stat(Db, <<"sizes">>, <<"external">>, AddSize - RemSize),
ok.
diff --git a/src/mango/src/mango_app.erl b/src/mango/src/mango_app.erl
index 7a0c39d..221d57d 100644
--- a/src/mango/src/mango_app.erl
+++ b/src/mango/src/mango_app.erl
@@ -15,6 +15,7 @@
-export([start/2, stop/1]).
start(_Type, StartArgs) ->
+ mango_jobs:set_timeout(),
mango_sup:start_link(StartArgs).
stop(_State) ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 22ff6a8..96d1fb5 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,9 +19,7 @@
]).
-export([
-%% view_cb/2,
handle_message/2,
- handle_all_docs_message/2,
composite_indexes/2,
choose_best_index/2
]).
@@ -140,7 +138,7 @@ index_args(#cursor{} = Cursor) ->
mango_json_bookmark:update_args(Bookmark, Args1).
-execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
+execute(#cursor{db = Db, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
Cursor = Cursor0#cursor{
user_fun = UserFun,
user_acc = UserAcc,
@@ -152,26 +150,21 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{ok, UserAcc};
_ ->
Args = index_args(Cursor),
- #cursor{opts = Opts} = Cursor,
- Result = case mango_idx:def(Idx) of
- all_docs ->
- CB = fun ?MODULE:handle_all_docs_message/2,
- % all_docs
- mango_fdb:query_all_docs(Db, CB, Cursor, Args);
- _ ->
- CB = fun ?MODULE:handle_message/2,
- % json index
- mango_fdb:query(Db, CB, Cursor, Args)
- end,
+ CB = fun ?MODULE:handle_message/2,
+ Result = mango_fdb:query(Db, CB, Cursor, Args),
case Result of
{ok, LastCursor} ->
NewBookmark = mango_json_bookmark:create(LastCursor),
Arg = {add_key, bookmark, NewBookmark},
- {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
- Stats0 = LastCursor#cursor.execution_stats,
- FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
- {ok, FinalUserAcc1};
+ #cursor{
+ opts = Opts,
+ execution_stats = Stats0,
+ user_acc = FinalUserAcc0
+ } = LastCursor,
+ {_Go, FinalUserAcc1} = UserFun(Arg, FinalUserAcc0),
+ FinalUserAcc2 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc1),
+ FinalUserAcc3 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc2),
+ {ok, FinalUserAcc3};
{error, Reason} ->
{error, Reason}
end
@@ -334,20 +327,6 @@ handle_message({error, Reason}, _Cursor) ->
{error, Reason}.
-handle_all_docs_message({row, Props}, Cursor) ->
- io:format("ALL DOCS ~p ~n", [Props]),
- case is_design_doc(Props) of
- true -> {ok, Cursor};
- false ->
- Doc = couch_util:get_value(doc, Props),
- Key = couch_util:get_value(key, Props),
-
- handle_message({doc, Key, Doc}, Cursor)
- end;
-handle_all_docs_message(Message, Cursor) ->
- handle_message(Message, Cursor).
-
-
handle_doc(#cursor{skip = S} = C, _) when S > 0 ->
{ok, C#cursor{skip = S - 1}};
handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 ->
@@ -362,16 +341,6 @@ handle_doc(#cursor{limit = L, execution_stats = Stats} = C, Doc) when L > 0 ->
handle_doc(C, _Doc) ->
{stop, C}.
-
-%%ddocid(Idx) ->
-%% case mango_idx:ddoc(Idx) of
-%% <<"_design/", Rest/binary>> ->
-%% Rest;
-%% Else ->
-%% Else
-%% end.
-
-
%%apply_opts([], Args) ->
%% Args;
%%apply_opts([{r, RStr} | Rest], Args) ->
@@ -485,11 +454,6 @@ match_doc(Selector, Doc, ExecutionStats) ->
end.
-is_design_doc(RowProps) ->
- case couch_util:get_value(id, RowProps) of
- <<"_design/", _/binary>> -> true;
- _ -> false
- end.
update_bookmark_keys(#cursor{limit = Limit} = Cursor, {Key, Props}) when Limit > 0 ->
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index edbb27f..9b5206e 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -18,7 +18,6 @@
-include("mango.hrl").
-include("mango_idx.hrl").
-include("mango_cursor.hrl").
--include("mango_idx_view.hrl").
-export([
@@ -30,8 +29,9 @@
set_update_seq/3,
remove_doc/3,
write_doc/3,
- query_all_docs/4,
- query/4
+ query/4,
+ base_fold_opts/1,
+ mango_idx_prefix/2
]).
@@ -118,90 +118,26 @@ write_doc(TxDb, DocId, IdxResults) ->
add_key(TxDb, MangoIdxPrefix, Results, DocId)
end, IdxResults).
-
-query_all_docs(Db, CallBack, Cursor, Args) ->
+query(Db, CallBack, Cursor, Args) ->
#cursor{
index = Idx
} = Cursor,
- Opts = args_to_fdb_opts(Args, Idx) ++ [{include_docs, true}],
- io:format("ALL DOC OPTS ~p ~n", [Opts]),
- fabric2_db:fold_docs(Db, CallBack, Cursor, Opts).
+ Mod = mango_idx:fdb_mod(Idx),
+ Mod:query(Db, CallBack, Cursor, Args).
-query(Db, CallBack, Cursor, Args) ->
- #cursor{
- index = Idx
- } = Cursor,
- MangoIdxPrefix = mango_idx_prefix(Db, Idx#idx.ddoc),
- fabric2_fdb:transactional(Db, fun (TxDb) ->
- Acc0 = #{
- cursor => Cursor,
- prefix => MangoIdxPrefix,
- db => TxDb,
- callback => CallBack
- },
-
- Opts = args_to_fdb_opts(Args, Idx),
- io:format("OPTS ~p ~n", [Opts]),
- try
- Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix, fun fold_cb/2, Acc0, Opts),
- #{
- cursor := Cursor1
- } = Acc1,
- {ok, Cursor1}
- catch
- throw:{stop, StopCursor} ->
- {ok, StopCursor}
- end
- end).
-
-
-args_to_fdb_opts(Args, Idx) ->
+base_fold_opts(Args) ->
#{
- start_key := StartKey,
- start_key_docid := StartKeyDocId,
- end_key := EndKey,
- end_key_docid := EndKeyDocId,
dir := Direction,
skip := Skip
} = Args,
- io:format("ARGS ~p ~n", [Args]),
- io:format("START ~p ~n End ~p ~n", [StartKey, EndKey]),
- Mod = mango_idx:fdb_mod(Idx),
-
- StartKeyOpts = Mod:start_key_opts(StartKey, StartKeyDocId),
- EndKeyOpts = Mod:end_key_opts(EndKey, EndKeyDocId),
-
[
{skip, Skip},
{dir, Direction},
{streaming_mode, want_all},
{restart_tx, true}
- ] ++ StartKeyOpts ++ EndKeyOpts.
-
-
-fold_cb({Key, Val}, Acc) ->
- #{
- prefix := MangoIdxPrefix,
- db := Db,
- callback := Callback,
- cursor := Cursor
-
- } = Acc,
- {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
- SortKeys = couch_views_encoding:decode(Val),
- {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]),
- JSONDoc = couch_doc:to_json_obj(Doc, []),
- io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
- case Callback({doc, SortKeys, JSONDoc}, Cursor) of
- {ok, Cursor1} ->
- Acc#{
- cursor := Cursor1
- };
- {stop, Cursor1} ->
- throw({stop, Cursor1})
- end.
+ ].
mango_idx_prefix(TxDb, Id) ->
diff --git a/src/mango/src/mango_fdb_special.erl b/src/mango/src/mango_fdb_special.erl
index e8fd6c1..ef55bc4 100644
--- a/src/mango/src/mango_fdb_special.erl
+++ b/src/mango/src/mango_fdb_special.erl
@@ -14,19 +14,73 @@
-module(mango_fdb_special).
-include_lib("couch/include/couch_db.hrl").
+-include("mango_cursor.hrl").
-export([
- start_key_opts/2,
- end_key_opts/2
+ query/4
]).
-start_key_opts(StartKey, _StartKeyDocId) ->
+
+query(Db, CallBack, Cursor, Args) ->
+ Acc = #{
+ cursor => Cursor,
+ callback => CallBack
+ },
+ Opts = args_to_fdb_opts(Args),
+ io:format("ALL DOC OPTS ~p ~n", [Opts]),
+ {ok, Acc1} = fabric2_db:fold_docs(Db, fun fold_cb/2, Acc, Opts),
+ {ok, maps:get(cursor, Acc1)}.
+
+
+args_to_fdb_opts(Args) ->
+ #{
+ start_key := StartKey,
+ end_key := EndKey
+ } = Args,
+ BaseOpts = mango_fdb:base_fold_opts(Args),
+ BaseOpts ++ [{include_docs, true}]
+ ++ start_key_opts(StartKey) ++ end_key_opts(EndKey).
+
+
+start_key_opts(StartKey) ->
[{start_key, fabric2_util:encode_all_doc_key(StartKey)}].
-end_key_opts(?MAX_STR, _EndKeyDocId) ->
+end_key_opts(?MAX_STR) ->
[];
-end_key_opts(EndKey, _EndKeyDocId) ->
+end_key_opts(EndKey) ->
[{end_key, fabric2_util:encode_all_doc_key(EndKey)}].
+
+
+fold_cb({row, Props}, Acc) ->
+ #{
+ cursor := Cursor,
+ callback := Callback
+ } = Acc,
+ io:format("ALL DOCS ~p ~n", [Props]),
+ case is_design_doc(Props) of
+ true ->
+ {ok, Acc};
+ false ->
+ Doc = couch_util:get_value(doc, Props),
+ Key = couch_util:get_value(key, Props),
+ {Go, Cursor1} = Callback({doc, Key, Doc}, Cursor),
+ {Go, Acc#{cursor := Cursor1}}
+ end;
+
+fold_cb(Message, Acc) ->
+ #{
+ cursor := Cursor,
+ callback := Callback
+ } = Acc,
+ {Go, Cursor1} = Callback(Message, Cursor),
+ {Go, Acc#{cursor := Cursor1}}.
+
+
+is_design_doc(RowProps) ->
+ case couch_util:get_value(id, RowProps) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end.
diff --git a/src/mango/src/mango_fdb_view.erl b/src/mango/src/mango_fdb_view.erl
index faab91b..329087f 100644
--- a/src/mango/src/mango_fdb_view.erl
+++ b/src/mango/src/mango_fdb_view.erl
@@ -15,10 +15,55 @@
-export([
- start_key_opts/2,
- end_key_opts/2
+ query/4
]).
+
+-include("mango_idx.hrl").
+-include("mango_cursor.hrl").
+
+
+query(Db, CallBack, Cursor, Args) ->
+ #cursor{
+ index = Idx
+ } = Cursor,
+ MangoIdxPrefix = mango_fdb:mango_idx_prefix(Db, Idx#idx.ddoc),
+ fabric2_fdb:transactional(Db, fun (TxDb) ->
+ Acc0 = #{
+ cursor => Cursor,
+ prefix => MangoIdxPrefix,
+ db => TxDb,
+ callback => CallBack
+ },
+
+ Opts = args_to_fdb_opts(Args),
+ io:format("OPTS ~p ~n", [Opts]),
+ try
+ Acc1 = fabric2_fdb:fold_range(TxDb, MangoIdxPrefix,
+ fun fold_cb/2, Acc0, Opts),
+ #{
+ cursor := Cursor1
+ } = Acc1,
+ {ok, Cursor1}
+ catch
+ throw:{stop, StopCursor} ->
+ {ok, StopCursor}
+ end
+ end).
+
+
+args_to_fdb_opts(Args) ->
+ #{
+ start_key := StartKey,
+ start_key_docid := StartKeyDocId,
+ end_key := EndKey,
+ end_key_docid := EndKeyDocId
+ } = Args,
+ BaseOpts = mango_fdb:base_fold_opts(Args),
+ BaseOpts ++ start_key_opts(StartKey, StartKeyDocId)
+ ++ end_key_opts(EndKey, EndKeyDocId).
+
+
start_key_opts([], _StartKeyDocId) ->
[];
@@ -32,6 +77,27 @@ end_key_opts([], _EndKeyDocId) ->
[];
end_key_opts(EndKey, EndKeyDocId) ->
- io:format("ENDKEY ~p ~n", [EndKey]),
EndKey1 = couch_views_encoding:encode(EndKey, key),
[{end_key, {EndKey1, EndKeyDocId}}].
+
+fold_cb({Key, Val}, Acc) ->
+ #{
+ prefix := MangoIdxPrefix,
+ db := Db,
+ callback := Callback,
+ cursor := Cursor
+
+ } = Acc,
+ {{_, DocId}} = erlfdb_tuple:unpack(Key, MangoIdxPrefix),
+ SortKeys = couch_views_encoding:decode(Val),
+ {ok, Doc} = fabric2_db:open_doc(Db, DocId, [{conflicts, true}]),
+ JSONDoc = couch_doc:to_json_obj(Doc, []),
+ io:format("PRINT ~p ~p ~n", [DocId, JSONDoc]),
+ case Callback({doc, SortKeys, JSONDoc}, Cursor) of
+ {ok, Cursor1} ->
+ Acc#{
+ cursor := Cursor1
+ };
+ {stop, Cursor1} ->
+ throw({stop, Cursor1})
+ end.
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index d00a254..b0b119f 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -66,8 +66,7 @@ doc_id(#doc{id = DocId}, _) ->
DocId.
-% Design doc
-% Todo: Check if design doc is mango index and kick off background worker
+% Check if design doc is mango index and kick off background worker
% to build new index
modify_int(Db, _Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc,
_PrevDoc) ->