You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2022/08/06 14:10:56 UTC
[couchdb] 10/21: feat(access): add mrview machinery
This is an automated email from the ASF dual-hosted git repository.
jan pushed a commit to branch feat/access-2022
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c0e639324833b033449ea3b24f8183dc56f5a62a
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sat Jun 25 11:28:53 2022 +0200
feat(access): add mrview machinery
---
src/couch_index/src/couch_index_updater.erl | 35 +++++---
src/couch_mrview/include/couch_mrview.hrl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 112 +++++++++++++++++++++++++-
src/couch_mrview/src/couch_mrview_updater.erl | 46 +++++++++--
src/couch_mrview/src/couch_mrview_util.erl | 9 ++-
5 files changed, 186 insertions(+), 19 deletions(-)
diff --git a/src/couch_index/src/couch_index_updater.erl b/src/couch_index/src/couch_index_updater.erl
index fe2150505..66d760622 100644
--- a/src/couch_index/src/couch_index_updater.erl
+++ b/src/couch_index/src/couch_index_updater.erl
@@ -123,8 +123,8 @@ update(Idx, Mod, IdxState) ->
IncludeDesign = lists:member(include_design, UpdateOpts),
DocOpts =
case lists:member(local_seq, UpdateOpts) of
- true -> [conflicts, deleted_conflicts, local_seq];
- _ -> [conflicts, deleted_conflicts]
+ true -> [conflicts, deleted_conflicts, local_seq, deleted];
+ _ -> [conflicts, deleted_conflicts,local_seq, deleted]
end,
couch_util:with_db(DbName, fun(Db) ->
@@ -142,23 +142,36 @@ update(Idx, Mod, IdxState) ->
end,
GetInfo = fun
- (#full_doc_info{id = Id, update_seq = Seq, deleted = Del} = FDI) ->
- {Id, Seq, Del, couch_doc:to_doc_info(FDI)};
- (#doc_info{id = Id, high_seq = Seq, revs = [RI | _]} = DI) ->
- {Id, Seq, RI#rev_info.deleted, DI}
+ (#full_doc_info{id=Id, update_seq=Seq, deleted=Del,access=Access}=FDI) ->
+ {Id, Seq, Del, couch_doc:to_doc_info(FDI), Access};
+ (#doc_info{id=Id, high_seq=Seq, revs=[RI|_],access=Access}=DI) ->
+ {Id, Seq, RI#rev_info.deleted, DI, Access}
end,
LoadDoc = fun(DI) ->
- {DocId, Seq, Deleted, DocInfo} = GetInfo(DI),
+ {DocId, Seq, Deleted, DocInfo, Access} = GetInfo(DI),
case {IncludeDesign, DocId} of
{false, <<"_design/", _/binary>>} ->
{nil, Seq};
- _ when Deleted ->
- {#doc{id = DocId, deleted = true}, Seq};
_ ->
- {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
- {Doc, Seq}
+ case IndexName of % TODO: move into outer case statement
+ <<"_design/_access">> ->
+ {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+ % TODO: hande conflicted docs in _access index
+ % probably remove
+ [RevInfo|_] = DocInfo#doc_info.revs,
+ Doc1 = Doc#doc{
+ meta = [{body_sp, RevInfo#rev_info.body_sp}],
+ access = Access
+ },
+ {Doc1, Seq};
+ _ when Deleted ->
+ {#doc{id=DocId, deleted=true}, Seq};
+ _ ->
+ {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+ {Doc, Seq}
+ end
end
end,
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index b31463c53..ef987595d 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -83,7 +83,8 @@
conflicts,
callback,
sorted = true,
- extra = []
+ extra = [],
+ deleted = false
}).
-record(vacc, {
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index d8640c903..79b2b8bec 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -13,7 +13,7 @@
-module(couch_mrview).
-export([validate/2]).
--export([query_all_docs/2, query_all_docs/4]).
+-export([query_all_docs/2, query_all_docs/4, query_changes_access/5]).
-export([query_view/3, query_view/4, query_view/6, get_view_index_pid/4]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
@@ -259,6 +259,116 @@ query_all_docs(Db, Args) ->
query_all_docs(Db, Args, Callback, Acc) when is_list(Args) ->
query_all_docs(Db, to_mrargs(Args), Callback, Acc);
query_all_docs(Db, Args0, Callback, Acc) ->
+ case couch_db:has_access_enabled(Db) and not couch_db:is_admin(Db) of
+ true -> query_all_docs_access(Db, Args0, Callback, Acc);
+ false -> query_all_docs_admin(Db, Args0, Callback, Acc)
+ end.
+access_ddoc() ->
+ #doc{
+ id = <<"_design/_access">>,
+ body = {[
+ {<<"language">>,<<"_access">>},
+ {<<"options">>, {[
+ {<<"include_design">>, true}
+ ]}},
+ {<<"views">>, {[
+ {<<"_access_by_id">>, {[
+ {<<"map">>, <<"_access/by-id-map">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}},
+ {<<"_access_by_seq">>, {[
+ {<<"map">>, <<"_access/by-seq-map">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}}
+ ]}}
+ ]}
+ }.
+query_changes_access(Db, StartSeq, Fun, Options, Acc) ->
+ DDoc = access_ddoc(),
+ UserCtx = couch_db:get_user_ctx(Db),
+ UserName = UserCtx#user_ctx.name,
+ %% % TODO: add roles
+ Args1 = prefix_startkey_endkey(UserName, #mrargs{}, fwd),
+ Args2 = Args1#mrargs{deleted=true},
+ Args = Args2#mrargs{reduce=false},
+ %% % filter out the user-prefix from the key, so _all_docs looks normal
+ %% % this isn’t a separate function because I’m binding Callback0 and I don’t
+ %% % know the Erlang equivalent of JS’s fun.bind(this, newarg)
+ Callback = fun
+ ({meta, _}, Acc0) ->
+ {ok, Acc0}; % ignore for now
+ ({row, Props}, Acc0) ->
+ % turn row into FDI
+ Value = couch_util:get_value(value, Props),
+ [Owner, Seq] = couch_util:get_value(key, Props),
+ Rev = couch_util:get_value(rev, Value),
+ Deleted = couch_util:get_value(deleted, Value, false),
+ BodySp = couch_util:get_value(body_sp, Value),
+ [Pos, RevId] = string:split(?b2l(Rev), "-"),
+ FDI = #full_doc_info{
+ id = proplists:get_value(id, Props),
+ rev_tree = [{list_to_integer(Pos), {?l2b(RevId), #leaf{deleted=Deleted, ptr=BodySp, seq=Seq, sizes=#size_info{}}, []}}],
+ deleted = Deleted,
+ update_seq = 0,
+ sizes = #size_info{},
+ access = [Owner]
+ },
+ Fun(FDI, Acc0);
+ (_Else, Acc0) ->
+ {ok, Acc0} % ignore for now
+ end,
+ VName = <<"_access_by_seq">>,
+ query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+query_all_docs_access(Db, Args0, Callback0, Acc) ->
+ % query our not yest existing, home-grown _access view.
+ % use query_view for this.
+ DDoc = access_ddoc(),
+ UserCtx = couch_db:get_user_ctx(Db),
+ UserName = UserCtx#user_ctx.name,
+ Args1 = prefix_startkey_endkey(UserName, Args0, Args0#mrargs.direction),
+ Args = Args1#mrargs{reduce=false, extra=Args1#mrargs.extra ++ [{all_docs_access, true}]},
+ Callback = fun
+ ({row, Props}, Acc0) ->
+ % filter out the user-prefix from the key, so _all_docs looks normal
+ % this isn’t a separate function because I’m binding Callback0 and I
+ % don’t know the Erlang equivalent of JS’s fun.bind(this, newarg)
+ [_User, Key] = proplists:get_value(key, Props),
+ Row0 = proplists:delete(key, Props),
+ Row = [{key, Key} | Row0],
+ Callback0({row, Row}, Acc0);
+ (Row, Acc0) ->
+ Callback0(Row, Acc0)
+ end,
+ VName = <<"_access_by_id">>,
+ query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+prefix_startkey_endkey(UserName, Args, fwd) ->
+ #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+ Args#mrargs {
+ start_key = case StartKey of
+ undefined -> [UserName];
+ StartKey -> [UserName, StartKey]
+ end,
+ end_key = case EndKey of
+ undefined -> [UserName, {}];
+ EndKey -> [UserName, EndKey, {}]
+ end
+ };
+
+prefix_startkey_endkey(UserName, Args, rev) ->
+ #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+ Args#mrargs {
+ end_key = case StartKey of
+ undefined -> [UserName];
+ StartKey -> [UserName, StartKey]
+ end,
+ start_key = case EndKey of
+ undefined -> [UserName, {}];
+ EndKey -> [UserName, EndKey, {}]
+ end
+ }.
+query_all_docs_admin(Db, Args0, Callback, Acc) ->
Sig = couch_util:with_db(Db, fun(WDb) ->
{ok, Info} = couch_db:get_db_info(WDb),
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info)))
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 969a82028..5d58ab05d 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -124,8 +124,9 @@ process_doc(Doc, Seq, #mrst{doc_acc = Acc} = State) when length(Acc) > 100 ->
process_doc(Doc, Seq, State#mrst{doc_acc = []});
process_doc(nil, Seq, #mrst{doc_acc = Acc} = State) ->
{ok, State#mrst{doc_acc = [{nil, Seq, nil} | Acc]}};
-process_doc(#doc{id = Id, deleted = true}, Seq, #mrst{doc_acc = Acc} = State) ->
- {ok, State#mrst{doc_acc = [{Id, Seq, deleted} | Acc]}};
+% TODO: re-evaluate why this is commented out
+% process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
+% {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
process_doc(#doc{id = Id} = Doc, Seq, #mrst{doc_acc = Acc} = State) ->
{ok, State#mrst{doc_acc = [{Id, Seq, Doc} | Acc]}}.
@@ -149,6 +150,14 @@ finish_update(#mrst{doc_acc = Acc} = State) ->
}}
end.
+make_deleted_body({Props}, Meta, Seq) ->
+ BodySp = couch_util:get_value(body_sp, Meta),
+ Result = [{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}],
+ case couch_util:get_value(<<"_access">>, Props) of
+ undefined -> Result;
+ Access -> [{<<"_access">>, Access} | Result]
+ end.
+
map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
erlang:put(io_priority, {view_update, DbName, IdxName}),
case couch_work_queue:dequeue(State0#mrst.doc_queue) of
@@ -167,11 +176,38 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
DocFun = fun
({nil, Seq, _}, {SeqAcc, Results}) ->
{erlang:max(Seq, SeqAcc), Results};
- ({Id, Seq, deleted}, {SeqAcc, Results}) ->
- {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
+ ({Id, Seq, Rev, #doc{deleted=true, body=Body, meta=Meta}}, {SeqAcc, Results}) ->
+ % _access needs deleted docs
+ case IdxName of
+ <<"_design/_access">> ->
+ % splice in seq
+ {Start, Rev1} = Rev,
+ Doc = #doc{
+ id = Id,
+ revs = {Start, [Rev1]},
+ body = {make_deleted_body(Body, Meta, Seq)}, %% todo: only keep _access and add _seq
+ deleted = true
+ },
+ {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, Res} | Results]};
+ _Else ->
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, []} | Results]}
+ end;
({Id, Seq, Doc}, {SeqAcc, Results}) ->
couch_stats:increment_counter([couchdb, mrview, map_doc]),
- {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+ % IdxName: ~p, Doc: ~p~n~n", [IdxName, Doc]),
+ Doc0 = case IdxName of
+ <<"_design/_access">> ->
+ % splice in seq
+ {Props} = Doc#doc.body,
+ BodySp = couch_util:get_value(body_sp, Doc#doc.meta),
+ Doc#doc{
+ body = {Props++[{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}]}
+ };
+ _Else ->
+ Doc
+ end,
+ {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc0),
{erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
end,
FoldFun = fun(Docs, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 9e3d292ed..cb90199a2 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -20,6 +20,7 @@
-export([index_file/2, compaction_file/2, open_file/1]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_access_row_count/2]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
@@ -384,6 +385,11 @@ reduce_to_count(Reductions) ->
FinalReduction = couch_btree:final_reduce(CountReduceFun, Reductions),
get_count(FinalReduction).
+get_access_row_count(#mrview{btree=Bt}, UserName) ->
+ couch_btree:full_reduce_with_options(Bt, [
+ {start_key, UserName}
+ ]).
+
fold(#mrview{btree = Bt}, Fun, Acc, Opts) ->
WrapperFun = fun(KV, Reds, Acc2) ->
fold_fun(Fun, expand_dups([KV], []), Reds, Acc2)
@@ -426,8 +432,9 @@ validate_args(#mrst{} = State, Args0) ->
ViewPartitioned = State#mrst.partitioned,
Partition = get_extra(Args, partition),
+ AllDocsAccess = get_extra(Args, all_docs_access, false),
- case {ViewPartitioned, Partition} of
+ case {ViewPartitioned and not AllDocsAccess, Partition} of
{true, undefined} ->
Msg1 = <<
"`partition` parameter is mandatory "