You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ei...@apache.org on 2019/11/22 06:08:42 UTC
[couchdb] 06/07: Remove all view_changes functionality from mrview
utils
This is an automated email from the ASF dual-hosted git repository.
eiri pushed a commit to branch 2167-no-view-changes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 46e17ad90c63eb1a1fc5b40c59e45f6808962a86
Author: Eric Avdey <ei...@eiri.ca>
AuthorDate: Fri Nov 22 01:52:15 2019 -0400
Remove all view_changes functionality from mrview utils
---
src/couch_mrview/src/couch_mrview_util.erl | 262 +++--------------------------
1 file changed, 24 insertions(+), 238 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index eb68124..c8bdfaf 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -20,7 +20,6 @@
-export([index_file/2, compaction_file/2, open_file/1]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
--export([get_view_changes_count/1]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
@@ -33,9 +32,6 @@
-export([get_view_keys/1, get_view_queries/1]).
-export([set_view_type/3]).
-export([set_extra/3, get_extra/2, get_extra/3]).
--export([changes_key_opts/2]).
--export([fold_changes/4]).
--export([to_key_seq/1]).
-define(MOD, couch_mrview_index).
-define(GET_VIEW_RETRY_COUNT, 1).
@@ -172,15 +168,13 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
DictBySrcAcc
end,
{DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}),
- SeqIndexed = proplists:get_value(<<"seq_indexed">>, DesignOpts, false),
- KeySeqIndexed = proplists:get_value(<<"keyseq_indexed">>, DesignOpts, false),
Partitioned = proplists:get_value(<<"partitioned">>, DesignOpts, false),
{RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
NumViews = fun({_, View}, N) ->
- {View#mrview{id_num=N, seq_indexed=SeqIndexed, keyseq_indexed=KeySeqIndexed}, N+1}
+ {View#mrview{id_num=N}, N+1}
end,
{Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
@@ -194,8 +188,6 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
views=Views,
language=Language,
design_opts=DesignOpts,
- seq_indexed=SeqIndexed,
- keyseq_indexed=KeySeqIndexed,
partitioned=Partitioned
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
@@ -253,11 +245,7 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
UpdateSeq = couch_db:get_update_seq(Db),
PurgeSeq = couch_db:get_purge_seq(Db),
- #mrst{
- seq_indexed=SeqIndexed,
- keyseq_indexed=KeySeqIndexed
- } = State,
- Term = view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed),
+ Term = view_sig_term(BaseSig, UpdateSeq, PurgeSeq),
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Term)));
view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
view_sig(Db, State, View, Args);
@@ -265,24 +253,18 @@ view_sig(_Db, State, View, Args0) ->
Sig = State#mrst.sig,
UpdateSeq = View#mrview.update_seq,
PurgeSeq = View#mrview.purge_seq,
- SeqIndexed = View#mrview.seq_indexed,
- KeySeqIndexed = View#mrview.keyseq_indexed,
Args = Args0#mrargs{
preflight_fun=undefined,
extra=[]
},
- Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args),
+ Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, Args),
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Term))).
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false) ->
- {BaseSig, UpdateSeq, PurgeSeq};
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed) ->
- {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed}.
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq) ->
+ {BaseSig, UpdateSeq, PurgeSeq}.
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false, Args) ->
- {BaseSig, UpdateSeq, PurgeSeq, Args};
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) ->
- {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}.
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, Args) ->
+ {BaseSig, UpdateSeq, PurgeSeq, Args}.
init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
@@ -291,7 +273,6 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
seq=0,
purge_seq=PurgeSeq,
id_btree_state=nil,
- log_btree_state=nil,
view_states=[make_view_state(#mrview{}) || _ <- Views]
},
init_state(Db, Fd, State, Header);
@@ -306,21 +287,17 @@ init_state(Db, Fd, State, #index_header{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=IdBtreeState,
- log_btree_state=nil,
view_states=[make_view_state(V) || V <- ViewStates]
});
init_state(Db, Fd, State, Header) ->
#mrst{
language=Lang,
- views=Views,
- seq_indexed=SeqIndexed,
- keyseq_indexed=KeySeqIndexed
+ views=Views
} = State,
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=IdBtreeState,
- log_btree_state=LogBtreeState,
view_states=ViewStates
} = Header,
@@ -328,10 +305,6 @@ init_state(Db, Fd, State, Header) ->
{compression, couch_compress:get_compression_method()}
],
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
- {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of
- true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
- false -> {ok, nil}
- end,
OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
Views2 = lists:zipwith(OpenViewFun, ViewStates, Views),
@@ -342,7 +315,6 @@ init_state(Db, Fd, State, Header) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
- log_btree=LogBtree,
views=Views2
}.
@@ -358,29 +330,7 @@ open_view(_Db, Fd, Lang, ViewState, View) ->
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- BySeqReduceFun = make_seq_reduce_fun(),
- {ok, SeqBtree} = if View#mrview.seq_indexed ->
- SeqBTState = get_seq_btree_state(ViewState),
- ViewSeqBtOpts = [{reduce, BySeqReduceFun},
- {compression, Compression}],
-
- couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);
- true ->
- {ok, nil}
- end,
- {ok, KeyBySeqBtree} = if View#mrview.keyseq_indexed ->
- KSeqBTState = get_kseq_btree_state(ViewState),
- KeyBySeqBtOpts = [{less, LessFun},
- {reduce, BySeqReduceFun},
- {compression, Compression}],
- couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts);
- true ->
- {ok, nil}
- end,
-
View#mrview{btree=Btree,
- seq_btree=SeqBtree,
- key_byseq_btree=KeyBySeqBtree,
update_seq=get_update_seq(ViewState),
purge_seq=get_purge_seq(ViewState)}.
@@ -424,26 +374,6 @@ reduce_to_count(Reductions) ->
FinalReduction = couch_btree:final_reduce(CountReduceFun, Reductions),
get_count(FinalReduction).
-%% @doc get all changes for a view
-get_view_changes_count(View) ->
- #mrview{seq_btree=SBtree, key_byseq_btree=KSBtree} = View,
- CountFun = fun(_SeqStart, PartialReds, 0) ->
- {ok, couch_btree:final_reduce(SBtree, PartialReds)}
- end,
- {ok, Count} = case {SBtree, KSBtree} of
- {nil, nil} ->
- {ok, 0};
- {#btree{}, nil} ->
- couch_btree:fold_reduce(SBtree, CountFun, 0, []);
- {_, #btree{}} ->
- couch_btree:fold_reduce(KSBtree, CountFun, 0, [])
- end,
- case {SBtree, KSBtree} of
- {#btree{}, #btree{}} ->
- {ok, Count*2};
- _ ->
- {ok, Count}
- end.
fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
WrapperFun = fun(KV, Reds, Acc2) ->
@@ -462,23 +392,6 @@ fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
end.
-fold_changes(Bt, Fun, Acc, Opts) ->
- WrapperFun = fun(KV, _Reds, Acc2) ->
- fold_changes_fun(Fun, changes_expand_dups([KV], []), Acc2)
- end,
- {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
-
-fold_changes_fun(_Fun, [], Acc) ->
- {ok, Acc};
-fold_changes_fun(Fun, [KV|Rest], Acc) ->
- case Fun(KV, Acc) of
- {ok, Acc2} ->
- fold_changes_fun(Fun, Rest, Acc2);
- {stop, Acc2} ->
- {stop, Acc2}
- end.
-
-
fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
#mrview{
btree=Bt,
@@ -812,7 +725,6 @@ make_header(State) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
- log_btree=LogBtree,
views=Views
} = State,
@@ -820,7 +732,6 @@ make_header(State) ->
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=get_btree_state(IdBtree),
- log_btree_state=get_btree_state(LogBtree),
view_states=[make_view_state(V) || V <- Views]
}.
@@ -876,16 +787,9 @@ reset_state(State) ->
State#mrst{
fd=nil,
qserver=nil,
- seq_indexed=State#mrst.seq_indexed,
- keyseq_indexed=State#mrst.keyseq_indexed,
update_seq=0,
id_btree=nil,
- log_btree=nil,
- views=[View#mrview{btree=nil, seq_btree=nil,
- key_byseq_btree=nil,
- seq_indexed=View#mrview.seq_indexed,
- keyseq_indexed=View#mrview.keyseq_indexed}
- || View <- State#mrst.views]
+ views=[View#mrview{btree=nil} || View <- State#mrst.views]
}.
@@ -953,45 +857,6 @@ reverse_key_default(<<255>>) -> <<>>;
reverse_key_default(Key) -> Key.
-changes_key_opts(StartSeq, Args) ->
- changes_key_opts(StartSeq, Args, []).
-
-
-changes_key_opts(StartSeq, #mrargs{keys=undefined, direction=Dir}=Args, Extra) ->
- [[{dir, Dir}] ++ changes_skey_opts(StartSeq, Args) ++
- changes_ekey_opts(StartSeq, Args) ++ Extra];
-changes_key_opts(StartSeq, #mrargs{keys=Keys, direction=Dir}=Args, Extra) ->
- lists:map(fun(K) ->
- [{dir, Dir}]
- ++ changes_skey_opts(StartSeq, Args#mrargs{start_key=K})
- ++ changes_ekey_opts(StartSeq, Args#mrargs{end_key=K})
- ++ Extra
- end, Keys).
-
-
-changes_skey_opts(StartSeq, #mrargs{start_key=undefined}) ->
- [{start_key, [<<>>, StartSeq+1]}];
-changes_skey_opts(StartSeq, #mrargs{start_key=SKey,
- start_key_docid=SKeyDocId}) ->
- [{start_key, {[SKey, StartSeq+1], SKeyDocId}}].
-
-
-changes_ekey_opts(_StartSeq, #mrargs{end_key=undefined}) ->
- [];
-changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
- end_key_docid=EKeyDocId,
- direction=Dir}=Args) ->
- EndSeq = case Dir of
- fwd -> 16#10000000;
- rev -> 0
- end,
-
- case Args#mrargs.inclusive_end of
- true -> [{end_key, {[EKey, EndSeq], EKeyDocId}}];
- false -> [{end_key_gt, {[EKey, EndSeq], EKeyDocId}}]
- end.
-
-
reduced_external_size(Tree) ->
case couch_btree:full_reduce(Tree) of
{ok, {_, _, Size}} -> Size;
@@ -1000,53 +865,26 @@ reduced_external_size(Tree) ->
end.
-reduced_seq_external_size(Tree) ->
- case couch_btree:full_reduce(Tree) of
- {ok, {_, Size}} -> Size;
- % return 0 for older versions that only returned number of docs
- {ok, NumDocs} when is_integer(NumDocs) -> 0
- end.
-
-
calculate_external_size(Views) ->
- SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
- Size0 = sum_btree_sizes(Acc, reduced_external_size(Bt)),
- Size1 = case SBt of
- nil -> Size0;
- _ -> sum_btree_sizes(Size0, reduced_seq_external_size(SBt))
- end,
- case KSBt of
- nil -> Size1;
- _ -> sum_btree_sizes(Size1, reduced_seq_external_size(KSBt))
- end
+ SumFun = fun
+ (#mrview{btree=nil}, Acc) ->
+ Acc;
+ (#mrview{btree=Bt}, Acc) ->
+ Acc + reduced_external_size(Bt)
end,
{ok, lists:foldl(SumFun, 0, Views)}.
calculate_active_size(Views) ->
- BtSize = fun
- (nil) -> 0;
- (Bt) -> couch_btree:size(Bt)
- end,
- FoldFun = fun(View, Acc) ->
- Sizes = [
- BtSize(View#mrview.btree),
- BtSize(View#mrview.seq_btree),
- BtSize(View#mrview.key_byseq_btree)
- ],
- Acc + lists:sum([S || S <- Sizes, is_integer(S)])
+ FoldFun = fun
+ (#mrview{btree=nil}, Acc) ->
+ Acc;
+ (#mrview{btree=Bt}, Acc) ->
+ Acc + couch_btree:size(Bt)
end,
{ok, lists:foldl(FoldFun, 0, Views)}.
-sum_btree_sizes(nil, _) ->
- 0;
-sum_btree_sizes(_, nil) ->
- 0;
-sum_btree_sizes(Size1, Size2) ->
- Size1 + Size2.
-
-
detuple_kvs([], Acc) ->
lists:reverse(Acc);
detuple_kvs([KV | Rest], Acc) ->
@@ -1064,19 +902,6 @@ expand_dups([KV | Rest], Acc) ->
expand_dups(Rest, [KV | Acc]).
-changes_expand_dups([], Acc) ->
- lists:reverse(Acc);
-changes_expand_dups([{{[Key, Seq], DocId}, {dups, Vals}} | Rest], Acc) ->
- Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
- changes_expand_dups(Rest, Expanded ++ Acc);
-changes_expand_dups([{{Seq, Key}, {DocId, {dups, Vals}}} | Rest], Acc) ->
- Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
- changes_expand_dups(Rest, Expanded ++ Acc);
-changes_expand_dups([{{[Key, Seq], DocId}, {Val, _}} | Rest], Acc) ->
- changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]);
-changes_expand_dups([{{Seq, Key}, {DocId, Val, _}} | Rest], Acc) ->
- changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]).
-
maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
[];
maybe_load_doc(Db, #doc_info{}=DI, #mrargs{conflicts=true, doc_options=Opts}) ->
@@ -1126,9 +951,6 @@ mrverror(Mesg) ->
throw({query_parse_error, Mesg}).
-to_key_seq(L) ->
- [{{[Key, Seq], DocId}, {Val, Rev}} || {{Seq, Key}, {DocId, Val, Rev}} <- L].
-
%% Updates 1.2.x or earlier view files to 1.3.x or later view files
%% transparently, the first time the 1.2.x view file is opened by
%% 1.3.x or later.
@@ -1217,45 +1039,25 @@ old_view_format(View) ->
make_view_state(#mrview{} = View) ->
BTState = get_btree_state(View#mrview.btree),
- SeqBTState = case View#mrview.seq_indexed of
- true ->
- get_btree_state(View#mrview.seq_btree);
- _ ->
- nil
- end,
- KSeqBTState = case View#mrview.keyseq_indexed of
- true ->
- get_btree_state(View#mrview.key_byseq_btree);
- _ ->
- nil
- end,
{
BTState,
- SeqBTState,
- KSeqBTState,
View#mrview.update_seq,
View#mrview.purge_seq
};
-make_view_state({BTState, UpdateSeq, PurgeSeq}) ->
- {BTState, nil, nil, UpdateSeq, PurgeSeq};
+make_view_state({BTState, _SeqBTState, _KSeqBTState, UpdateSeq, PurgeSeq}) ->
+ {BTState, UpdateSeq, PurgeSeq};
make_view_state(nil) ->
- {nil, nil, nil, 0, 0}.
+ {nil, 0, 0}.
get_key_btree_state(ViewState) ->
element(1, ViewState).
-get_seq_btree_state(ViewState) ->
- element(2, ViewState).
-
-get_kseq_btree_state(ViewState) ->
- element(3, ViewState).
-
get_update_seq(ViewState) ->
- element(4, ViewState).
+ element(2, ViewState).
get_purge_seq(ViewState) ->
- element(5, ViewState).
+ element(3, ViewState).
get_count(Reduction) ->
element(1, Reduction).
@@ -1296,22 +1098,6 @@ make_reduce_fun(Lang, ReduceFuns) ->
{Counts, Result, ExternalSize}
end.
-make_seq_reduce_fun() ->
- fun
- (reduce, KVs0) ->
- KVs = detuple_kvs(expand_dups(KVs0, []), []),
- NumDocs = length(KVs),
- ExternalSize = kv_external_size(KVs, NumDocs),
- {NumDocs, ExternalSize};
- (rereduce, Reds) ->
- ExtractFun = fun(Red, {NumDocsAcc0, ExtAcc0}) ->
- NumDocsAcc = NumDocsAcc0 + get_count(Red),
- ExtAcc = ExtAcc0 + get_external_size_reds(Red),
- {NumDocsAcc, ExtAcc}
- end,
- lists:foldl(ExtractFun, {0, 0}, Reds)
- end.
-
maybe_define_less_fun(#mrview{options = Options}) ->
case couch_util:get_value(<<"collation">>, Options) of