You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2020/07/26 18:10:13 UTC
[couchdb] 11/17: feat(access): use access by-seq/by-id for regular
users
This is an automated email from the ASF dual-hosted git repository.
jan pushed a commit to branch feat/access-master-clean
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e34cad849981d7247f1f5eff4862d2f785804ce1
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Sun Jul 26 20:04:33 2020 +0200
feat(access): use access by-seq/by-id for regular users
---
src/couch/src/couch_btree.erl | 12 +++
src/couch_index/src/couch_index_updater.erl | 35 +++++---
src/couch_mrview/include/couch_mrview.hrl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 119 +++++++++++++++++++++++++-
src/couch_mrview/src/couch_mrview_updater.erl | 45 ++++++++--
src/couch_mrview/src/couch_mrview_util.erl | 8 +-
6 files changed, 203 insertions(+), 19 deletions(-)
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index ea0cf69..75e801e 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -16,6 +16,7 @@
-export([fold/4, full_reduce/1, final_reduce/2, size/1, foldl/3, foldl/4]).
-export([fold_reduce/4, lookup/2, get_state/1, set_options/2]).
-export([extract/2, assemble/3, less/3]).
+-export([full_reduce_with_options/2]).
-include_lib("couch/include/couch_db.hrl").
@@ -92,6 +93,17 @@ fold_reduce(#btree{root=Root}=Bt, Fun, Acc, Options) ->
throw:{stop, AccDone} -> {ok, AccDone}
end.
+full_reduce_with_options(Bt, Options0) ->
+ CountFun = fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(Bt, PartialReds)}
+ end,
+ [UserName] = proplists:get_value(start_key, Options0, <<"">>),
+ EndKey = {[UserName, {[]}]},
+ Options = Options0 ++ [
+ {end_key, EndKey}
+ ],
+ fold_reduce(Bt, CountFun, 0, Options).
+
full_reduce(#btree{root=nil,reduce=Reduce}) ->
{ok, Reduce(reduce, [])};
full_reduce(#btree{root=Root}) ->
diff --git a/src/couch_index/src/couch_index_updater.erl b/src/couch_index/src/couch_index_updater.erl
index fb15db0..2f65c1c 100644
--- a/src/couch_index/src/couch_index_updater.erl
+++ b/src/couch_index/src/couch_index_updater.erl
@@ -135,8 +135,8 @@ update(Idx, Mod, IdxState) ->
CommittedOnly = lists:member(committed_only, UpdateOpts),
IncludeDesign = lists:member(include_design, UpdateOpts),
DocOpts = case lists:member(local_seq, UpdateOpts) of
- true -> [conflicts, deleted_conflicts, local_seq];
- _ -> [conflicts, deleted_conflicts]
+ true -> [conflicts, deleted_conflicts, local_seq, deleted];
+ _ -> [conflicts, deleted_conflicts, local_seq, deleted]
end,
couch_util:with_db(DbName, fun(Db) ->
@@ -154,23 +154,36 @@ update(Idx, Mod, IdxState) ->
end,
GetInfo = fun
- (#full_doc_info{id=Id, update_seq=Seq, deleted=Del}=FDI) ->
- {Id, Seq, Del, couch_doc:to_doc_info(FDI)};
- (#doc_info{id=Id, high_seq=Seq, revs=[RI|_]}=DI) ->
- {Id, Seq, RI#rev_info.deleted, DI}
+ (#full_doc_info{id=Id, update_seq=Seq, deleted=Del,access=Access}=FDI) ->
+ {Id, Seq, Del, couch_doc:to_doc_info(FDI), Access};
+ (#doc_info{id=Id, high_seq=Seq, revs=[RI|_],access=Access}=DI) ->
+ {Id, Seq, RI#rev_info.deleted, DI, Access}
end,
LoadDoc = fun(DI) ->
- {DocId, Seq, Deleted, DocInfo} = GetInfo(DI),
+ {DocId, Seq, Deleted, DocInfo, Access} = GetInfo(DI),
case {IncludeDesign, DocId} of
{false, <<"_design/", _/binary>>} ->
{nil, Seq};
- _ when Deleted ->
- {#doc{id=DocId, deleted=true}, Seq};
_ ->
- {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
- {Doc, Seq}
+ case IndexName of % TODO: move into outer case statement
+ <<"_design/_access">> ->
+ {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+ % TODO: hande conflicted docs in _access index
+ % probably remove
+ [RevInfo|_] = DocInfo#doc_info.revs,
+ Doc1 = Doc#doc{
+ meta = [{body_sp, RevInfo#rev_info.body_sp}],
+ access = Access
+ },
+ {Doc1, Seq};
+ _ when Deleted ->
+ {#doc{id=DocId, deleted=true}, Seq};
+ _ ->
+ {ok, Doc} = couch_db:open_doc_int(Db, DocInfo, DocOpts),
+ {Doc, Seq}
+ end
end
end,
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index bb0ab0b..d9d7a8b 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -81,7 +81,8 @@
conflicts,
callback,
sorted = true,
- extra = []
+ extra = [],
+ deleted = false
}).
-record(vacc, {
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 1cdc918..298576d 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -13,7 +13,7 @@
-module(couch_mrview).
-export([validate/2]).
--export([query_all_docs/2, query_all_docs/4]).
+-export([query_all_docs/2, query_all_docs/4, query_changes_access/5]).
-export([query_view/3, query_view/4, query_view/6, get_view_index_pid/4]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
@@ -233,6 +233,123 @@ query_all_docs(Db, Args) ->
query_all_docs(Db, Args, Callback, Acc) when is_list(Args) ->
query_all_docs(Db, to_mrargs(Args), Callback, Acc);
query_all_docs(Db, Args0, Callback, Acc) ->
+ case couch_db:has_access_enabled(Db) and not couch_db:is_admin(Db) of
+ true -> query_all_docs_access(Db, Args0, Callback, Acc);
+ false -> query_all_docs_admin(Db, Args0, Callback, Acc)
+ end.
+
+access_ddoc() ->
+ #doc{
+ id = <<"_design/_access">>,
+ body = {[
+ {<<"language">>,<<"_access">>},
+ {<<"options">>, {[
+ {<<"include_design">>, true}
+ ]}},
+ {<<"views">>, {[
+ {<<"_access_by_id">>, {[
+ {<<"map">>, <<"_access/by-id-map">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}},
+ {<<"_access_by_seq">>, {[
+ {<<"map">>, <<"_access/by-seq-map">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}}
+ ]}}
+ ]}
+ }.
+
+query_changes_access(Db, StartSeq, Fun, Options, Acc) ->
+ DDoc = access_ddoc(),
+ UserCtx = couch_db:get_user_ctx(Db),
+ UserName = UserCtx#user_ctx.name,
+ %% % TODO: add roles
+ Args1 = prefix_startkey_endkey(UserName, #mrargs{}, fwd),
+ Args2 = Args1#mrargs{deleted=true},
+ Args = Args2#mrargs{reduce=false},
+ %% % filter out the user-prefix from the key, so _all_docs looks normal
+ %% % this isn’t a separate function because I’m binding Callback0 and I don’t
+ %% % know the Erlang equivalent of JS’s fun.bind(this, newarg)
+ Callback = fun
+ ({meta, _}, Acc0) ->
+ {ok, Acc0}; % ignore for now
+ ({row, Props}, Acc0) ->
+ % turn row into FDI
+ Value = couch_util:get_value(value, Props),
+ [Owner, Seq] = couch_util:get_value(key, Props),
+
+ Rev = couch_util:get_value(rev, Value),
+ Deleted = couch_util:get_value(deleted, Value, false),
+ BodySp = couch_util:get_value(body_sp, Value),
+
+ [Pos, RevId] = string:split(?b2l(Rev), "-"),
+ FDI = #full_doc_info{
+ id = proplists:get_value(id, Props),
+ rev_tree = [{list_to_integer(Pos), {?l2b(RevId), #leaf{deleted=Deleted, ptr=BodySp, seq=Seq, sizes=#size_info{}}, []}}],
+ deleted = Deleted,
+ update_seq = 0,
+ sizes = #size_info{},
+ access = [Owner]
+ },
+ Fun(FDI, Acc0);
+ (_Else, Acc0) ->
+ {ok, Acc0} % ignore for now
+ end,
+ VName = <<"_access_by_seq">>,
+ query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+query_all_docs_access(Db, Args0, Callback0, Acc) ->
+ % query our not yest existing, home-grown _access view.
+ % use query_view for this.
+ DDoc = access_ddoc(),
+ UserCtx = couch_db:get_user_ctx(Db),
+ UserName = UserCtx#user_ctx.name,
+ Args1 = prefix_startkey_endkey(UserName, Args0, Args0#mrargs.direction),
+ Args = Args1#mrargs{reduce=false, extra=Args1#mrargs.extra ++ [{all_docs_access, true}]},
+
+ Callback = fun
+ ({row, Props}, Acc0) ->
+
+ % filter out the user-prefix from the key, so _all_docs looks normal
+ % this isn’t a separate function because I’m binding Callback0 and I
+ % don’t know the Erlang equivalent of JS’s fun.bind(this, newarg)
+ [_User, Key] = proplists:get_value(key, Props),
+ Row0 = proplists:delete(key, Props),
+ Row = [{key, Key} | Row0],
+
+ Callback0({row, Row}, Acc0);
+ (Row, Acc0) ->
+ Callback0(Row, Acc0)
+ end,
+ VName = <<"_access_by_id">>,
+ query_view(Db, DDoc, VName, Args, Callback, Acc).
+
+prefix_startkey_endkey(UserName, Args, fwd) ->
+ #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+ Args#mrargs {
+ start_key = case StartKey of
+ undefined -> [UserName];
+ StartKey -> [UserName, StartKey]
+ end,
+ end_key = case EndKey of
+ undefined -> [UserName, {}];
+ EndKey -> [UserName, EndKey, {}]
+ end
+ };
+prefix_startkey_endkey(UserName, Args, rev) ->
+ #mrargs{start_key=StartKey, end_key=EndKey} = Args,
+ Args#mrargs {
+ end_key = case StartKey of
+ undefined -> [UserName];
+ StartKey -> [UserName, StartKey]
+ end,
+ start_key = case EndKey of
+ undefined -> [UserName, {}];
+ EndKey -> [UserName, EndKey, {}]
+ end
+ }.
+
+query_all_docs_admin(Db, Args0, Callback, Acc) ->
Sig = couch_util:with_db(Db, fun(WDb) ->
{ok, Info} = couch_db:get_db_info(WDb),
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info)))
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 522367c..0dc9f85 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -116,8 +116,9 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
process_doc(Doc, Seq, State#mrst{doc_acc=[]});
process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
{ok, State#mrst{doc_acc=[{nil, Seq, nil} | Acc]}};
-process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
- {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
+% TODO: re-evaluate why this is commented out
+% process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
+% {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
{ok, State#mrst{doc_acc=[{Id, Seq, Doc} | Acc]}}.
@@ -140,6 +141,13 @@ finish_update(#mrst{doc_acc=Acc}=State) ->
}}
end.
+make_deleted_body({Props}, Meta, Seq) ->
+ BodySp = couch_util:get_value(body_sp, Meta),
+ Result = [{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}],
+ case couch_util:get_value(<<"_access">>, Props) of
+ undefined -> Result;
+ Access -> [{<<"_access">>, Access} | Result]
+ end.
map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
erlang:put(io_priority, {view_update, DbName, IdxName}),
@@ -158,11 +166,38 @@ map_docs(Parent, #mrst{db_name = DbName, idx_name = IdxName} = State0) ->
DocFun = fun
({nil, Seq, _}, {SeqAcc, Results}) ->
{erlang:max(Seq, SeqAcc), Results};
- ({Id, Seq, deleted}, {SeqAcc, Results}) ->
- {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
+ ({Id, Seq, Rev, #doc{deleted=true, body=Body, meta=Meta}}, {SeqAcc, Results}) ->
+ % _access needs deleted docs
+ case IdxName of
+ <<"_design/_access">> ->
+ % splice in seq
+ {Start, Rev1} = Rev,
+ Doc = #doc{
+ id = Id,
+ revs = {Start, [Rev1]},
+ body = {make_deleted_body(Body, Meta, Seq)}, %% todo: only keep _access and add _seq
+ deleted = true
+ },
+ {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, Res} | Results]};
+ _Else ->
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, []} | Results]}
+ end;
({Id, Seq, Doc}, {SeqAcc, Results}) ->
couch_stats:increment_counter([couchdb, mrview, map_doc]),
- {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
+ % IdxName: ~p, Doc: ~p~n~n", [IdxName, Doc]),
+ Doc0 = case IdxName of
+ <<"_design/_access">> ->
+ % splice in seq
+ {Props} = Doc#doc.body,
+ BodySp = couch_util:get_value(body_sp, Doc#doc.meta),
+ Doc#doc{
+ body = {Props++[{<<"_seq">>, Seq}, {<<"_body_sp">>, BodySp}]}
+ };
+ _Else ->
+ Doc
+ end,
+ {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc0),
{erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
end,
FoldFun = fun(Docs, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index e971720..698adb6 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -20,6 +20,7 @@
-export([index_file/2, compaction_file/2, open_file/1]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_access_row_count/2]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
@@ -340,6 +341,10 @@ temp_view_to_ddoc({Props}) ->
]},
couch_doc:from_json_obj(DDoc).
+get_access_row_count(#mrview{btree=Bt}, UserName) ->
+ couch_btree:full_reduce_with_options(Bt, [
+ {start_key, UserName}
+ ]).
get_row_count(#mrview{btree=Bt}) ->
Count = case couch_btree:full_reduce(Bt) of
@@ -407,8 +412,9 @@ validate_args(#mrst{} = State, Args0) ->
ViewPartitioned = State#mrst.partitioned,
Partition = get_extra(Args, partition),
+ AllDocsAccess = get_extra(Args, all_docs_access, false),
- case {ViewPartitioned, Partition} of
+ case {ViewPartitioned and not AllDocsAccess, Partition} of
{true, undefined} ->
Msg1 = <<"`partition` parameter is mandatory "
"for queries to this view.">>,