You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2019/08/30 18:21:02 UTC
[couchdb] branch
prototype/fdb-layer-add-queries-support-for-all-docs created (now 1140e68)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch prototype/fdb-layer-add-queries-support-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 1140e68 Implement _all_docs/queries
This branch includes the following new commits:
new 1140e68 Implement _all_docs/queries
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Implement _all_docs/queries
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch prototype/fdb-layer-add-queries-support-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1140e686b7707cc23ea8a4d477f211ae897f96d2
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Fri Aug 30 14:16:31 2019 -0400
Implement _all_docs/queries
Also add mrargs validation to match what master does and provide some
helpful feedback to the users.
---
src/chttpd/src/chttpd_db.erl | 193 +++++++++++++++++--------------
src/couch_views/src/couch_views_util.erl | 145 ++++++++++++++++++++++-
test/elixir/test/basics_test.exs | 39 +++++++
3 files changed, 289 insertions(+), 88 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 42cbb7d..1837585 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -805,113 +805,132 @@ db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) ->
db_attachment_req(Req, Db, DocId, FileNameParts).
multi_all_docs_view(Req, Db, OP, Queries) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, undefined),
Args1 = Args0#mrargs{view_type=map},
ArgQueries = lists:map(fun({Query}) ->
QueryArg1 = couch_mrview_http:parse_params(Query, undefined,
Args1, [decoded]),
- QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1),
+ QueryArgs2 = couch_views_util:validate_args(QueryArg1),
set_namespace(OP, QueryArgs2)
end, Queries),
- Options = [{user_ctx, Req#httpd.user_ctx}],
- VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
- FirstChunk = "{\"results\":[",
- {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req,
- 200, [], FirstChunk),
- VAcc1 = VAcc0#vacc{resp=Resp0},
- VAcc2 = lists:foldl(fun(Args, Acc0) ->
- {ok, Acc1} = fabric2_db:fold_docs(Db, Options,
- fun view_cb/2, Acc0, Args),
- Acc1
- end, VAcc1, ArgQueries),
- {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+ Max = chttpd:chunked_response_buffer_size(),
+ First = "{\"results\":[",
+ {ok, Resp0} = chttpd:start_delayed_json_response(Req, 200, [], First),
+ VAcc0 = #vacc{
+ db = Db,
+ req = Req,
+ resp = Resp0,
+ threshold = Max,
+ prepend = "\r\n"
+ },
+ VAcc1 = lists:foldl(fun
+ (#mrargs{keys = undefined} = Args, Acc0) ->
+ send_all_docs(Db, Args, UserCtx, Acc0);
+ (#mrargs{keys = Keys} = Args, Acc0) when is_list(Keys) ->
+ send_all_docs_keys(Db, Args, UserCtx, Acc0)
+ end, VAcc0, ArgQueries),
+ {ok, Resp1} = chttpd:send_delayed_chunk(VAcc1#vacc.resp, "\r\n]}"),
chttpd:end_delayed_json_response(Resp1).
+
all_docs_view(Req, Db, Keys, OP) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, Keys),
- Args1 = set_namespace(OP, Args0),
+ Args1 = Args0#mrargs{view_type=map},
+ Args2 = couch_views_util:validate_args(Args1),
+ Args3 = set_namespace(OP, Args2),
Max = chttpd:chunked_response_buffer_size(),
VAcc0 = #vacc{
db = Db,
req = Req,
threshold = Max
},
- case Args1#mrargs.keys of
+ case Args3#mrargs.keys of
undefined ->
- Options = all_docs_view_opts(Args1, Req),
- Acc = {iter, Db, Args1, VAcc0},
- {ok, {iter, _, _, Resp}} =
- fabric2_db:fold_docs(Db, fun view_cb/2, Acc, Options),
- {ok, Resp#vacc.resp};
- Keys0 when is_list(Keys0) ->
- Keys1 = apply_args_to_keylist(Args1, Keys0),
- %% namespace can be _set_ to `undefined`, so we
- %% want simulate enum here
- NS = case couch_util:get_value(namespace, Args1#mrargs.extra) of
- <<"_all_docs">> -> <<"_all_docs">>;
- <<"_design">> -> <<"_design">>;
- <<"_local">> -> <<"_local">>;
- _ -> <<"_all_docs">>
- end,
- TotalRows = fabric2_db:get_doc_count(Db, NS),
- Meta = case Args1#mrargs.update_seq of
- true ->
- UpdateSeq = fabric2_db:get_update_seq(Db),
- [{update_seq, UpdateSeq}];
- false ->
- []
- end ++ [{total, TotalRows}, {offset, null}],
- {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
- DocOpts = case Args1#mrargs.conflicts of
- true -> [conflicts | Args1#mrargs.doc_options];
- _ -> Args1#mrargs.doc_options
- end ++ [{user_ctx, Req#httpd.user_ctx}],
- IncludeDocs = Args1#mrargs.include_docs,
- VAcc2 = lists:foldl(fun(DocId, Acc) ->
- OpenOpts = [deleted | DocOpts],
- Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
- {not_found, missing} ->
- #view_row{key = DocId};
- {ok, #doc{deleted = true, revs = Revs}} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})},
- {deleted, true}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- null
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- };
- {ok, #doc{revs = Revs} = Doc0} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- couch_doc:to_json_obj(Doc0, DocOpts)
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- }
- end,
- Row1 = fabric_view:transform_row(Row0),
- {ok, NewAcc} = view_cb(Row1, Acc),
- NewAcc
- end, VAcc1, Keys1),
- {ok, VAcc3} = view_cb(complete, VAcc2),
- {ok, VAcc3#vacc.resp}
+ VAcc1 = send_all_docs(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc1#vacc.resp};
+ Keys when is_list(Keys) ->
+ VAcc1 = send_all_docs_keys(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc2} = view_cb(complete, VAcc1),
+ {ok, VAcc2#vacc.resp}
end.
-all_docs_view_opts(Args, Req) ->
+send_all_docs(Db, #mrargs{keys = undefined} = Args, UserCtx, VAcc0) ->
+ Opts = all_docs_view_opts(Args, UserCtx),
+ Acc = {iter, Db, Args, VAcc0},
+ ViewCb = fun view_cb/2,
+ {ok, {iter, _, _, VAcc1}} = fabric2_db:fold_docs(Db, ViewCb, Acc, Opts),
+ VAcc1.
+
+
+send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
+ Keys = apply_args_to_keylist(Args, Args#mrargs.keys),
+ %% namespace can be _set_ to `undefined`, so we
+ %% want simulate enum here
+ NS = case couch_util:get_value(namespace, Args#mrargs.extra) of
+ <<"_all_docs">> -> <<"_all_docs">>;
+ <<"_design">> -> <<"_design">>;
+ <<"_local">> -> <<"_local">>;
+ _ -> <<"_all_docs">>
+ end,
+ TotalRows = fabric2_db:get_doc_count(Db, NS),
+ Meta = case Args#mrargs.update_seq of
+ true ->
+ UpdateSeq = fabric2_db:get_update_seq(Db),
+ [{update_seq, UpdateSeq}];
+ false ->
+ []
+ end ++ [{total, TotalRows}, {offset, null}],
+ {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
+ DocOpts = case Args#mrargs.conflicts of
+ true -> [conflicts | Args#mrargs.doc_options];
+ _ -> Args#mrargs.doc_options
+ end ++ [{user_ctx, UserCtx}],
+ IncludeDocs = Args#mrargs.include_docs,
+ VAcc2 = lists:foldl(fun(DocId, Acc) ->
+ OpenOpts = [deleted | DocOpts],
+ Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ #view_row{key = DocId};
+ {ok, #doc{deleted = true, revs = Revs}} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})},
+ {deleted, true}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ null
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ };
+ {ok, #doc{revs = Revs} = Doc0} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ couch_doc:to_json_obj(Doc0, DocOpts)
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ }
+ end,
+ Row1 = fabric_view:transform_row(Row0),
+ {ok, NewAcc} = view_cb(Row1, Acc),
+ NewAcc
+ end, VAcc1, Keys).
+
+
+all_docs_view_opts(Args, UserCtx) ->
StartKey = case Args#mrargs.start_key of
undefined -> Args#mrargs.start_key_docid;
SKey -> SKey
@@ -925,7 +944,7 @@ all_docs_view_opts(Args, Req) ->
{_, _} -> [{end_key, EndKey}]
end,
[
- {user_ctx, Req#httpd.user_ctx},
+ {user_ctx, UserCtx},
{dir, Args#mrargs.direction},
{start_key, StartKey},
{limit, Args#mrargs.limit},
diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl
index b88cfcd..9be68e4 100644
--- a/src/couch_views/src/couch_views_util.erl
+++ b/src/couch_views/src/couch_views_util.erl
@@ -14,7 +14,8 @@
-export([
- ddoc_to_mrst/2
+ ddoc_to_mrst/2,
+ validate_args/1
]).
@@ -82,3 +83,145 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}.
+
+
+% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't
+% update start / end keys and also throws a not_implemented error for reduce
+%
+validate_args(#mrargs{} = Args) ->
+ GroupLevel = determine_group_level(Args),
+ Reduce = Args#mrargs.reduce,
+
+ case Reduce =/= undefined orelse Args#mrargs.view_type == red of
+ true -> throw(not_supported);
+ false -> ok
+ end,
+
+ case Reduce == undefined orelse is_boolean(Reduce) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid `reduce` value.">>)
+ end,
+
+ case {Args#mrargs.view_type, Reduce} of
+ {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of
+ {red, exact, _} -> ok;
+ {red, _, KeyList} when is_list(KeyList) ->
+ Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>,
+ mrverror(Msg);
+ _ -> ok
+ end,
+
+ case Args#mrargs.keys of
+ Keys when is_list(Keys) -> ok;
+ undefined -> ok;
+ _ -> mrverror(<<"`keys` must be an array of strings.">>)
+ end,
+
+ case {Args#mrargs.keys, Args#mrargs.start_key,
+ Args#mrargs.end_key} of
+ {undefined, _, _} -> ok;
+ {[], _, _} -> ok;
+ {[_|_], undefined, undefined} -> ok;
+ _ -> mrverror(<<"`keys` is incompatible with `key`"
+ ", `start_key` and `end_key`">>)
+ end,
+
+ case Args#mrargs.start_key_docid of
+ undefined -> ok;
+ SKDocId0 when is_binary(SKDocId0) -> ok;
+ _ -> mrverror(<<"`start_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.end_key_docid of
+ undefined -> ok;
+ EKDocId0 when is_binary(EKDocId0) -> ok;
+ _ -> mrverror(<<"`end_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.direction of
+ fwd -> ok;
+ rev -> ok;
+ _ -> mrverror(<<"Invalid direction.">>)
+ end,
+
+ case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of
+ {true, _} -> ok;
+ {_, true} -> ok;
+ _ -> mrverror(<<"`limit` must be a positive integer.">>)
+ end,
+
+ case Args#mrargs.skip < 0 of
+ true -> mrverror(<<"`skip` must be >= 0">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel} of
+ {red, exact} -> ok;
+ {_, 0} -> ok;
+ {red, Int} when is_integer(Int), Int >= 0 -> ok;
+ {red, _} -> mrverror(<<"`group_level` must be >= 0">>);
+ {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>)
+ end,
+
+ case Args#mrargs.stable of
+ true -> ok;
+ false -> ok;
+ _ -> mrverror(<<"Invalid value for `stable`.">>)
+ end,
+
+ case Args#mrargs.update of
+ true -> ok;
+ false -> ok;
+ lazy -> ok;
+ _ -> mrverror(<<"Invalid value for `update`.">>)
+ end,
+
+ case is_boolean(Args#mrargs.inclusive_end) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `inclusive_end`.">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.include_docs} of
+ {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>);
+ {_, ID} when is_boolean(ID) -> ok;
+ _ -> mrverror(<<"Invalid value for `include_docs`">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.conflicts} of
+ {_, undefined} -> ok;
+ {map, V} when is_boolean(V) -> ok;
+ {red, undefined} -> ok;
+ {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>);
+ {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>)
+ end,
+
+ case is_boolean(Args#mrargs.sorted) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `sorted`.">>)
+ end,
+
+ Args#mrargs{group_level=GroupLevel}.
+
+
+determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 ->
+ mrverror(<<"Can't specify group=false and group_level>0 at the same time">>);
+
+determine_group_level(#mrargs{group=true, group_level=undefined}) ->
+ exact;
+
+determine_group_level(#mrargs{group_level=GroupLevel}) ->
+ GroupLevel.
+
+
+mrverror(Mesg) ->
+ throw({query_parse_error, Mesg}).
diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs
index 363972b..c32fec9 100644
--- a/test/elixir/test/basics_test.exs
+++ b/test/elixir/test/basics_test.exs
@@ -316,4 +316,43 @@ defmodule BasicsTest do
# TODO
assert true
end
+
+ @tag :with_db
+ test "POST _all_docs/queries works POST", context do
+ db_name = context[:db_name]
+
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: %{:queries => []})
+ assert resp.status_code == 200
+ assert resp.body["results"] == []
+
+ assert Couch.put("/#{db_name}/doc1", body: %{:a => 1}).body["ok"]
+
+ body = %{
+ :queries => [
+ %{:limit => 1},
+ %{:limit => 0}
+ ]
+ }
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: body)
+ assert resp.status_code == 200
+ assert Map.has_key?(resp.body, "results")
+ results = Enum.sort(resp.body["results"])
+ assert length(results) == 2
+
+ [res1, res2] = results
+
+ assert res1 == %{"offset" => :null, "rows" => [], "total_rows" => 1}
+
+ assert res2["offset"] == :null
+ assert res2["total_rows"] == 1
+
+ rows = res2["rows"]
+ assert length(rows) == 1
+ [row] = rows
+ assert row["id"] == "doc1"
+ assert row["key"] == "doc1"
+ val = row["value"]
+ assert Map.has_key?(val, "rev")
+ end
+
end