You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2019/08/30 18:41:12 UTC
[couchdb] branch prototype/fdb-layer updated: Implement
_all_docs/queries
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/prototype/fdb-layer by this push:
new d5d5b5f Implement _all_docs/queries
d5d5b5f is described below
commit d5d5b5fae586197f8c8f024ccf1034ec8cbe7061
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Fri Aug 30 14:16:31 2019 -0400
Implement _all_docs/queries
Also add mrargs validation to match what master does and provide some
helpful feedback to the users.
---
src/chttpd/src/chttpd_db.erl | 193 +++++++++++++++++--------------
src/couch_views/src/couch_views_util.erl | 145 ++++++++++++++++++++++-
test/elixir/test/basics_test.exs | 40 +++++++
3 files changed, 290 insertions(+), 88 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 42cbb7d..1837585 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -805,113 +805,132 @@ db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) ->
db_attachment_req(Req, Db, DocId, FileNameParts).
multi_all_docs_view(Req, Db, OP, Queries) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, undefined),
Args1 = Args0#mrargs{view_type=map},
ArgQueries = lists:map(fun({Query}) ->
QueryArg1 = couch_mrview_http:parse_params(Query, undefined,
Args1, [decoded]),
- QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1),
+ QueryArgs2 = couch_views_util:validate_args(QueryArg1),
set_namespace(OP, QueryArgs2)
end, Queries),
- Options = [{user_ctx, Req#httpd.user_ctx}],
- VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
- FirstChunk = "{\"results\":[",
- {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req,
- 200, [], FirstChunk),
- VAcc1 = VAcc0#vacc{resp=Resp0},
- VAcc2 = lists:foldl(fun(Args, Acc0) ->
- {ok, Acc1} = fabric2_db:fold_docs(Db, Options,
- fun view_cb/2, Acc0, Args),
- Acc1
- end, VAcc1, ArgQueries),
- {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+ Max = chttpd:chunked_response_buffer_size(),
+ First = "{\"results\":[",
+ {ok, Resp0} = chttpd:start_delayed_json_response(Req, 200, [], First),
+ VAcc0 = #vacc{
+ db = Db,
+ req = Req,
+ resp = Resp0,
+ threshold = Max,
+ prepend = "\r\n"
+ },
+ VAcc1 = lists:foldl(fun
+ (#mrargs{keys = undefined} = Args, Acc0) ->
+ send_all_docs(Db, Args, UserCtx, Acc0);
+ (#mrargs{keys = Keys} = Args, Acc0) when is_list(Keys) ->
+ send_all_docs_keys(Db, Args, UserCtx, Acc0)
+ end, VAcc0, ArgQueries),
+ {ok, Resp1} = chttpd:send_delayed_chunk(VAcc1#vacc.resp, "\r\n]}"),
chttpd:end_delayed_json_response(Resp1).
+
all_docs_view(Req, Db, Keys, OP) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, Keys),
- Args1 = set_namespace(OP, Args0),
+ Args1 = Args0#mrargs{view_type=map},
+ Args2 = couch_views_util:validate_args(Args1),
+ Args3 = set_namespace(OP, Args2),
Max = chttpd:chunked_response_buffer_size(),
VAcc0 = #vacc{
db = Db,
req = Req,
threshold = Max
},
- case Args1#mrargs.keys of
+ case Args3#mrargs.keys of
undefined ->
- Options = all_docs_view_opts(Args1, Req),
- Acc = {iter, Db, Args1, VAcc0},
- {ok, {iter, _, _, Resp}} =
- fabric2_db:fold_docs(Db, fun view_cb/2, Acc, Options),
- {ok, Resp#vacc.resp};
- Keys0 when is_list(Keys0) ->
- Keys1 = apply_args_to_keylist(Args1, Keys0),
- %% namespace can be _set_ to `undefined`, so we
- %% want simulate enum here
- NS = case couch_util:get_value(namespace, Args1#mrargs.extra) of
- <<"_all_docs">> -> <<"_all_docs">>;
- <<"_design">> -> <<"_design">>;
- <<"_local">> -> <<"_local">>;
- _ -> <<"_all_docs">>
- end,
- TotalRows = fabric2_db:get_doc_count(Db, NS),
- Meta = case Args1#mrargs.update_seq of
- true ->
- UpdateSeq = fabric2_db:get_update_seq(Db),
- [{update_seq, UpdateSeq}];
- false ->
- []
- end ++ [{total, TotalRows}, {offset, null}],
- {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
- DocOpts = case Args1#mrargs.conflicts of
- true -> [conflicts | Args1#mrargs.doc_options];
- _ -> Args1#mrargs.doc_options
- end ++ [{user_ctx, Req#httpd.user_ctx}],
- IncludeDocs = Args1#mrargs.include_docs,
- VAcc2 = lists:foldl(fun(DocId, Acc) ->
- OpenOpts = [deleted | DocOpts],
- Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
- {not_found, missing} ->
- #view_row{key = DocId};
- {ok, #doc{deleted = true, revs = Revs}} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})},
- {deleted, true}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- null
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- };
- {ok, #doc{revs = Revs} = Doc0} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- couch_doc:to_json_obj(Doc0, DocOpts)
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- }
- end,
- Row1 = fabric_view:transform_row(Row0),
- {ok, NewAcc} = view_cb(Row1, Acc),
- NewAcc
- end, VAcc1, Keys1),
- {ok, VAcc3} = view_cb(complete, VAcc2),
- {ok, VAcc3#vacc.resp}
+ VAcc1 = send_all_docs(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc1#vacc.resp};
+ Keys when is_list(Keys) ->
+ VAcc1 = send_all_docs_keys(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc2} = view_cb(complete, VAcc1),
+ {ok, VAcc2#vacc.resp}
end.
-all_docs_view_opts(Args, Req) ->
+send_all_docs(Db, #mrargs{keys = undefined} = Args, UserCtx, VAcc0) ->
+ Opts = all_docs_view_opts(Args, UserCtx),
+ Acc = {iter, Db, Args, VAcc0},
+ ViewCb = fun view_cb/2,
+ {ok, {iter, _, _, VAcc1}} = fabric2_db:fold_docs(Db, ViewCb, Acc, Opts),
+ VAcc1.
+
+
+send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
+ Keys = apply_args_to_keylist(Args, Args#mrargs.keys),
+ %% namespace can be _set_ to `undefined`, so we
+ %% want simulate enum here
+ NS = case couch_util:get_value(namespace, Args#mrargs.extra) of
+ <<"_all_docs">> -> <<"_all_docs">>;
+ <<"_design">> -> <<"_design">>;
+ <<"_local">> -> <<"_local">>;
+ _ -> <<"_all_docs">>
+ end,
+ TotalRows = fabric2_db:get_doc_count(Db, NS),
+ Meta = case Args#mrargs.update_seq of
+ true ->
+ UpdateSeq = fabric2_db:get_update_seq(Db),
+ [{update_seq, UpdateSeq}];
+ false ->
+ []
+ end ++ [{total, TotalRows}, {offset, null}],
+ {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
+ DocOpts = case Args#mrargs.conflicts of
+ true -> [conflicts | Args#mrargs.doc_options];
+ _ -> Args#mrargs.doc_options
+ end ++ [{user_ctx, UserCtx}],
+ IncludeDocs = Args#mrargs.include_docs,
+ VAcc2 = lists:foldl(fun(DocId, Acc) ->
+ OpenOpts = [deleted | DocOpts],
+ Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ #view_row{key = DocId};
+ {ok, #doc{deleted = true, revs = Revs}} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})},
+ {deleted, true}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ null
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ };
+ {ok, #doc{revs = Revs} = Doc0} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ couch_doc:to_json_obj(Doc0, DocOpts)
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ }
+ end,
+ Row1 = fabric_view:transform_row(Row0),
+ {ok, NewAcc} = view_cb(Row1, Acc),
+ NewAcc
+ end, VAcc1, Keys).
+
+
+all_docs_view_opts(Args, UserCtx) ->
StartKey = case Args#mrargs.start_key of
undefined -> Args#mrargs.start_key_docid;
SKey -> SKey
@@ -925,7 +944,7 @@ all_docs_view_opts(Args, Req) ->
{_, _} -> [{end_key, EndKey}]
end,
[
- {user_ctx, Req#httpd.user_ctx},
+ {user_ctx, UserCtx},
{dir, Args#mrargs.direction},
{start_key, StartKey},
{limit, Args#mrargs.limit},
diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl
index b88cfcd..b9dbb71 100644
--- a/src/couch_views/src/couch_views_util.erl
+++ b/src/couch_views/src/couch_views_util.erl
@@ -14,7 +14,8 @@
-export([
- ddoc_to_mrst/2
+ ddoc_to_mrst/2,
+ validate_args/1
]).
@@ -82,3 +83,145 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}.
+
+
+% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't
+% update start / end keys and also throws a not_implemented error for reduce
+%
+validate_args(#mrargs{} = Args) ->
+ GroupLevel = determine_group_level(Args),
+ Reduce = Args#mrargs.reduce,
+
+ case Reduce =/= undefined orelse Args#mrargs.view_type == red of
+ true -> throw(not_implemented);
+ false -> ok
+ end,
+
+ case Reduce == undefined orelse is_boolean(Reduce) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid `reduce` value.">>)
+ end,
+
+ case {Args#mrargs.view_type, Reduce} of
+ {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of
+ {red, exact, _} -> ok;
+ {red, _, KeyList} when is_list(KeyList) ->
+ Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>,
+ mrverror(Msg);
+ _ -> ok
+ end,
+
+ case Args#mrargs.keys of
+ Keys when is_list(Keys) -> ok;
+ undefined -> ok;
+ _ -> mrverror(<<"`keys` must be an array of strings.">>)
+ end,
+
+ case {Args#mrargs.keys, Args#mrargs.start_key,
+ Args#mrargs.end_key} of
+ {undefined, _, _} -> ok;
+ {[], _, _} -> ok;
+ {[_|_], undefined, undefined} -> ok;
+ _ -> mrverror(<<"`keys` is incompatible with `key`"
+ ", `start_key` and `end_key`">>)
+ end,
+
+ case Args#mrargs.start_key_docid of
+ undefined -> ok;
+ SKDocId0 when is_binary(SKDocId0) -> ok;
+ _ -> mrverror(<<"`start_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.end_key_docid of
+ undefined -> ok;
+ EKDocId0 when is_binary(EKDocId0) -> ok;
+ _ -> mrverror(<<"`end_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.direction of
+ fwd -> ok;
+ rev -> ok;
+ _ -> mrverror(<<"Invalid direction.">>)
+ end,
+
+ case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of
+ {true, _} -> ok;
+ {_, true} -> ok;
+ _ -> mrverror(<<"`limit` must be a positive integer.">>)
+ end,
+
+ case Args#mrargs.skip < 0 of
+ true -> mrverror(<<"`skip` must be >= 0">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel} of
+ {red, exact} -> ok;
+ {_, 0} -> ok;
+ {red, Int} when is_integer(Int), Int >= 0 -> ok;
+ {red, _} -> mrverror(<<"`group_level` must be >= 0">>);
+ {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>)
+ end,
+
+ case Args#mrargs.stable of
+ true -> ok;
+ false -> ok;
+ _ -> mrverror(<<"Invalid value for `stable`.">>)
+ end,
+
+ case Args#mrargs.update of
+ true -> ok;
+ false -> ok;
+ lazy -> ok;
+ _ -> mrverror(<<"Invalid value for `update`.">>)
+ end,
+
+ case is_boolean(Args#mrargs.inclusive_end) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `inclusive_end`.">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.include_docs} of
+ {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>);
+ {_, ID} when is_boolean(ID) -> ok;
+ _ -> mrverror(<<"Invalid value for `include_docs`">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.conflicts} of
+ {_, undefined} -> ok;
+ {map, V} when is_boolean(V) -> ok;
+ {red, undefined} -> ok;
+ {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>);
+ {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>)
+ end,
+
+ case is_boolean(Args#mrargs.sorted) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `sorted`.">>)
+ end,
+
+ Args#mrargs{group_level=GroupLevel}.
+
+
+determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 ->
+ mrverror(<<"Can't specify group=false and group_level>0 at the same time">>);
+
+determine_group_level(#mrargs{group=true, group_level=undefined}) ->
+ exact;
+
+determine_group_level(#mrargs{group_level=GroupLevel}) ->
+ GroupLevel.
+
+
+mrverror(Mesg) ->
+ throw({query_parse_error, Mesg}).
diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs
index 363972b..70dd6e8 100644
--- a/test/elixir/test/basics_test.exs
+++ b/test/elixir/test/basics_test.exs
@@ -316,4 +316,44 @@ defmodule BasicsTest do
# TODO
assert true
end
+
+ @tag :with_db
+ test "_all_docs/queries works", context do
+ db_name = context[:db_name]
+
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: %{:queries => []})
+ assert resp.status_code == 200
+ assert resp.body["results"] == []
+
+ assert Couch.put("/#{db_name}/doc1", body: %{:a => 1}).body["ok"]
+
+ body = %{
+ :queries => [
+ %{:limit => 1},
+ %{:limit => 0}
+ ]
+ }
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: body)
+ assert resp.status_code == 200
+
+ assert Map.has_key?(resp.body, "results")
+ results = Enum.sort(resp.body["results"])
+ assert length(results) == 2
+ [res1, res2] = results
+
+ assert res1 == %{"offset" => :null, "rows" => [], "total_rows" => 1}
+
+ assert res2["offset"] == :null
+ assert res2["total_rows"] == 1
+ rows = res2["rows"]
+
+ assert length(rows) == 1
+ [row] = rows
+ assert row["id"] == "doc1"
+ assert row["key"] == "doc1"
+
+ val = row["value"]
+ assert Map.has_key?(val, "rev")
+ end
+
end