You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/07/29 22:19:57 UTC
[couchdb] 06/08: Use ebtree for reduce functions
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-ebtree-views
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3e05908d9e148c335b4c8105e9157085318268ad
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Jul 29 10:34:48 2020 -0500
Use ebtree for reduce functions
---
src/couch_views/src/couch_views.erl | 4 --
src/couch_views/src/couch_views_fdb.erl | 87 ++++++++++++++++++++++++++++
src/couch_views/src/couch_views_reader.erl | 92 +++++++++++++++++++++++++++---
3 files changed, 172 insertions(+), 11 deletions(-)
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index f6e163a..eea7c89 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -49,10 +49,6 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
Args2 = couch_mrview_util:set_view_type(Args1, ViewName, Views),
Args3 = couch_mrview_util:validate_args(Args2),
ok = check_range(Args3),
- case is_reduce_view(Args3) of
- true -> throw(not_implemented);
- false -> ok
- end,
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 3b81d5f..a5d07ca 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -241,6 +241,93 @@ fold_map_idx(TxDb, View, Options, Callback, Acc0) ->
end.
+fold_red_idx(TxDb, View, Idx, Language, Options, Callback, Acc0) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ #mrview{
+ btree = Btree
+ } = View,
+
+ CollateFun = collate_fun(View),
+
+ Dir = case lists:keyfind(dir, 1, Options) of
+ {dir, D} -> D;
+ _ -> fwd
+ end,
+
+ InclusiveEnd = case lists:keyfind(inclusive_end, 1, Options) of
+ {inclusive_end, IE} -> IE;
+ _ -> true
+ end,
+
+ StartKey = case lists:keyfind(start_key, 1, Options) of
+ {start_key, SK} -> SK;
+ false when Dir == fwd -> ebtree:min();
+ false when Dir == rev -> ebtree:max()
+ end,
+
+ EndKey = case lists:keyfind(end_key, 1, Options) of
+ {end_key, EK} -> EK;
+ false when Dir == fwd -> ebtree:max();
+ false when Dir == rev -> ebtree:min()
+ end,
+
+ Wrapper = fun(KVs0, WAcc) ->
+ % Remove any keys that match Start or End key
+ % depending on direction
+ KVs1 = case InclusiveEnd of
+ true ->
+ KVs0;
+ false when Dir == fwd ->
+ lists:filter(fun({K, _V}) ->
+ case CollateFun(K, EndKey) of
+ true ->
+ % K =< EndKey
+ case CollateFun(EndKey, K) of
+ true ->
+ % K == EndKey, so reject
+ false;
+ false ->
+ % K < EndKey, so include
+ true
+ end;
+ false when Dir == fwd ->
+ % K > EndKey, should never happen, but reject
+ false
+ end
+ end, KVs0);
+ false when Dir == rev ->
+ lists:filter(fun({K, _V}) ->
+ % In reverse, if K =< EndKey, we drop it
+ not CollateFun(K, EndKey)
+ end, KVs0)
+ end,
+ % Expand dups
+ KVs2 = lists:flatmap(fun({K, V}) ->
+ case V of
+ {dups, Dups} when Dir == fwd ->
+ [{K, D} || D <- Dups];
+ {dups, Dups} when Dir == rev ->
+ [{K, D} || D <- lists:reverse(Dups)];
+ _ ->
+ [{K, V}]
+ end
+ end, KVs1),
+ lists:foldl(fun({{Key, DocId}, Value}, WAccInner) ->
+ Callback(DocId, Key, Value, WAccInner)
+ end, WAcc, KVs2)
+ end,
+
+ case Dir of
+ fwd ->
+ ebtree:range(Tx, Btree, StartKey, EndKey, Wrapper, Acc0);
+ rev ->
+ % Start/End keys swapped on purpose because ebtree
+ ebtree:reverse_range(Tx, Btree, EndKey, StartKey, Wrapper, Acc0)
+ end.
+
+
write_doc(TxDb, Mrst, #{deleted := true} = Doc) ->
#{
tx := Tx
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 545b91a..e460245 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -23,7 +23,15 @@
-include_lib("fabric/include/fabric2.hrl").
-read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
+read(Db, Mrst, ViewName, UserCallback, UserAcc, Args) ->
+ ReadFun = case Args of
+ #mrargs{view_type = map} -> fun read_map_view/6;
+ #mrargs{view_type = red} -> fun read_red_view/6
+ end,
+ ReadFun(Db, Mrst, ViewName, UserCallback, UserAcc, Args).
+
+
+read_map_view(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
#mrst{
@@ -31,10 +39,10 @@ read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
views = Views
} = Mrst = couch_views_fdb:set_trees(TxDb, Mrst0),
- View = get_view(Lang, Args, ViewName, Views),
+ View = get_map_view(Lang, Args, ViewName, Views),
Fun = fun handle_row/4,
- Meta = get_meta(TxDb, Mrst, View, Args),
+ Meta = get_map_meta(TxDb, Mrst, View, Args),
UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
Acc0 = #{
@@ -68,16 +76,79 @@ read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
end.
-get_meta(TxDb, Mrst, View, #mrargs{update_seq = true}) ->
+read_red_view(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
+ try
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ #mrst{
+ language = Lang,
+ views = Views
+ } = Mrst = couch_views_fdb:set_trees(TxDb, Mrst0),
+
+ {Idx, Lang, View} = get_red_view(Lang, Args, ViewName, Views),
+ Fun = fun handle_row/4,
+
+ Meta = get_red_meta(TxDb, Mrst, View, Args),
+ UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
+
+ Acc0 = #{
+ db => TxDb,
+ skip => Args#mrargs.skip,
+ limit => Args#mrargs.limit,
+ mrargs => undefined,
+ red_idx => Idx,
+ language => Lang,
+ callback => UserCallback,
+ acc => UserAcc1
+ },
+
+ Acc1 = lists:foldl(fun(KeyArgs, KeyAcc0) ->
+ Opts = mrargs_to_fdb_options(KeyArgs),
+ KeyAcc1 = KeyAcc0#{
+ mrargs := KeyArgs
+ },
+ couch_views_fdb:fold_red_idx(
+ TxDb,
+ View,
+ Idx,
+ Lang,
+ Opts,
+ Fun,
+ KeyAcc1
+ )
+ end, Acc0, expand_keys_args(Args)),
+
+ #{
+ acc := UserAcc2
+ } = Acc1,
+ {ok, maybe_stop(UserCallback(complete, UserAcc2))}
+ end)
+ catch
+ throw:{complete, Out} ->
+ {_, Final} = UserCallback(complete, Out),
+ {ok, Final};
+ throw:{done, Out} ->
+ {ok, Out}
+ end.
+
+
+get_map_meta(TxDb, Mrst, View, #mrargs{update_seq = true}) ->
TotalRows = couch_views_fdb:get_row_count(TxDb, View),
ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
{meta, [{update_seq, ViewSeq}, {total, TotalRows}, {offset, null}]};
-get_meta(TxDb, _Mrst, View, #mrargs{}) ->
+get_map_meta(TxDb, _Mrst, View, #mrargs{}) ->
TotalRows = couch_views_fdb:get_row_count(TxDb, View),
{meta, [{total, TotalRows}, {offset, null}]}.
+get_red_meta(TxDb, Mrst, _View, #mrargs{update_seq = true}) ->
+ ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
+ {meta, [{update_seq, ViewSeq}]};
+
+get_red_meta(_TxDb, _Mrst, _View, #mrargs{}) ->
+ {meta, []}.
+
+
handle_row(_DocId, _Key, _Value, #{skip := Skip} = Acc) when Skip > 0 ->
Acc#{skip := Skip - 1};
@@ -115,10 +186,17 @@ handle_row(DocId, Key, Value, Acc) ->
Acc#{limit := Limit - 1, acc := UserAcc1}.
-get_view(Lang, Args, ViewName, Views) ->
+get_map_view(Lang, Args, ViewName, Views) ->
case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
{map, View, _Args} -> View;
- {red, {_Idx, _Lang, View}} -> View
+ {red, {Idx, Lang, View} = RedView} -> RedView
+ end.
+
+
+get_red_view(Lang, Args, ViewName, Views) ->
+ case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
+ {red, {Idx, Lang, View}} -> {Idx, Lang, View};
+ _ -> throw({not_found, missing_named_view})
end.