You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2019/08/30 18:27:31 UTC
[couchdb] branch
prototype/fdb-layer-add-queries-support-for-all-docs updated (0ee98c5 ->
06544b2)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch prototype/fdb-layer-add-queries-support-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 0ee98c5 Implement _all_docs/queries
new 06544b2 Implement _all_docs/queries
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0ee98c5)
\
N -- N -- N refs/heads/prototype/fdb-layer-add-queries-support-for-all-docs (06544b2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch_views/src/couch_views_util.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[couchdb] 01/01: Implement _all_docs/queries
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch prototype/fdb-layer-add-queries-support-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 06544b2da795dcdcc19ac2ca4c473f1793fc8765
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Fri Aug 30 14:16:31 2019 -0400
Implement _all_docs/queries
Also add mrargs validation to match what master does and provide some
helpful feedback to the users.
---
src/chttpd/src/chttpd_db.erl | 193 +++++++++++++++++--------------
src/couch_views/src/couch_views_util.erl | 145 ++++++++++++++++++++++-
test/elixir/test/basics_test.exs | 40 +++++++
3 files changed, 290 insertions(+), 88 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 42cbb7d..1837585 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -805,113 +805,132 @@ db_req(#httpd{path_parts=[_, DocId | FileNameParts]}=Req, Db) ->
db_attachment_req(Req, Db, DocId, FileNameParts).
multi_all_docs_view(Req, Db, OP, Queries) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, undefined),
Args1 = Args0#mrargs{view_type=map},
ArgQueries = lists:map(fun({Query}) ->
QueryArg1 = couch_mrview_http:parse_params(Query, undefined,
Args1, [decoded]),
- QueryArgs2 = fabric_util:validate_all_docs_args(Db, QueryArg1),
+ QueryArgs2 = couch_views_util:validate_args(QueryArg1),
set_namespace(OP, QueryArgs2)
end, Queries),
- Options = [{user_ctx, Req#httpd.user_ctx}],
- VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
- FirstChunk = "{\"results\":[",
- {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req,
- 200, [], FirstChunk),
- VAcc1 = VAcc0#vacc{resp=Resp0},
- VAcc2 = lists:foldl(fun(Args, Acc0) ->
- {ok, Acc1} = fabric2_db:fold_docs(Db, Options,
- fun view_cb/2, Acc0, Args),
- Acc1
- end, VAcc1, ArgQueries),
- {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+ Max = chttpd:chunked_response_buffer_size(),
+ First = "{\"results\":[",
+ {ok, Resp0} = chttpd:start_delayed_json_response(Req, 200, [], First),
+ VAcc0 = #vacc{
+ db = Db,
+ req = Req,
+ resp = Resp0,
+ threshold = Max,
+ prepend = "\r\n"
+ },
+ VAcc1 = lists:foldl(fun
+ (#mrargs{keys = undefined} = Args, Acc0) ->
+ send_all_docs(Db, Args, UserCtx, Acc0);
+ (#mrargs{keys = Keys} = Args, Acc0) when is_list(Keys) ->
+ send_all_docs_keys(Db, Args, UserCtx, Acc0)
+ end, VAcc0, ArgQueries),
+ {ok, Resp1} = chttpd:send_delayed_chunk(VAcc1#vacc.resp, "\r\n]}"),
chttpd:end_delayed_json_response(Resp1).
+
all_docs_view(Req, Db, Keys, OP) ->
+ UserCtx = Req#httpd.user_ctx,
Args0 = couch_mrview_http:parse_params(Req, Keys),
- Args1 = set_namespace(OP, Args0),
+ Args1 = Args0#mrargs{view_type=map},
+ Args2 = couch_views_util:validate_args(Args1),
+ Args3 = set_namespace(OP, Args2),
Max = chttpd:chunked_response_buffer_size(),
VAcc0 = #vacc{
db = Db,
req = Req,
threshold = Max
},
- case Args1#mrargs.keys of
+ case Args3#mrargs.keys of
undefined ->
- Options = all_docs_view_opts(Args1, Req),
- Acc = {iter, Db, Args1, VAcc0},
- {ok, {iter, _, _, Resp}} =
- fabric2_db:fold_docs(Db, fun view_cb/2, Acc, Options),
- {ok, Resp#vacc.resp};
- Keys0 when is_list(Keys0) ->
- Keys1 = apply_args_to_keylist(Args1, Keys0),
- %% namespace can be _set_ to `undefined`, so we
- %% want simulate enum here
- NS = case couch_util:get_value(namespace, Args1#mrargs.extra) of
- <<"_all_docs">> -> <<"_all_docs">>;
- <<"_design">> -> <<"_design">>;
- <<"_local">> -> <<"_local">>;
- _ -> <<"_all_docs">>
- end,
- TotalRows = fabric2_db:get_doc_count(Db, NS),
- Meta = case Args1#mrargs.update_seq of
- true ->
- UpdateSeq = fabric2_db:get_update_seq(Db),
- [{update_seq, UpdateSeq}];
- false ->
- []
- end ++ [{total, TotalRows}, {offset, null}],
- {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
- DocOpts = case Args1#mrargs.conflicts of
- true -> [conflicts | Args1#mrargs.doc_options];
- _ -> Args1#mrargs.doc_options
- end ++ [{user_ctx, Req#httpd.user_ctx}],
- IncludeDocs = Args1#mrargs.include_docs,
- VAcc2 = lists:foldl(fun(DocId, Acc) ->
- OpenOpts = [deleted | DocOpts],
- Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
- {not_found, missing} ->
- #view_row{key = DocId};
- {ok, #doc{deleted = true, revs = Revs}} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})},
- {deleted, true}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- null
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- };
- {ok, #doc{revs = Revs} = Doc0} ->
- {RevPos, [RevId | _]} = Revs,
- Value = {[
- {rev, couch_doc:rev_to_str({RevPos, RevId})}
- ]},
- DocValue = if not IncludeDocs -> undefined; true ->
- couch_doc:to_json_obj(Doc0, DocOpts)
- end,
- #view_row{
- key = DocId,
- id = DocId,
- value = Value,
- doc = DocValue
- }
- end,
- Row1 = fabric_view:transform_row(Row0),
- {ok, NewAcc} = view_cb(Row1, Acc),
- NewAcc
- end, VAcc1, Keys1),
- {ok, VAcc3} = view_cb(complete, VAcc2),
- {ok, VAcc3#vacc.resp}
+ VAcc1 = send_all_docs(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc1#vacc.resp};
+ Keys when is_list(Keys) ->
+ VAcc1 = send_all_docs_keys(Db, Args3, UserCtx, VAcc0),
+ {ok, VAcc2} = view_cb(complete, VAcc1),
+ {ok, VAcc2#vacc.resp}
end.
-all_docs_view_opts(Args, Req) ->
+send_all_docs(Db, #mrargs{keys = undefined} = Args, UserCtx, VAcc0) ->
+ Opts = all_docs_view_opts(Args, UserCtx),
+ Acc = {iter, Db, Args, VAcc0},
+ ViewCb = fun view_cb/2,
+ {ok, {iter, _, _, VAcc1}} = fabric2_db:fold_docs(Db, ViewCb, Acc, Opts),
+ VAcc1.
+
+
+send_all_docs_keys(Db, #mrargs{} = Args, UserCtx, VAcc0) ->
+ Keys = apply_args_to_keylist(Args, Args#mrargs.keys),
+ %% namespace can be _set_ to `undefined`, so we
+ %% want simulate enum here
+ NS = case couch_util:get_value(namespace, Args#mrargs.extra) of
+ <<"_all_docs">> -> <<"_all_docs">>;
+ <<"_design">> -> <<"_design">>;
+ <<"_local">> -> <<"_local">>;
+ _ -> <<"_all_docs">>
+ end,
+ TotalRows = fabric2_db:get_doc_count(Db, NS),
+ Meta = case Args#mrargs.update_seq of
+ true ->
+ UpdateSeq = fabric2_db:get_update_seq(Db),
+ [{update_seq, UpdateSeq}];
+ false ->
+ []
+ end ++ [{total, TotalRows}, {offset, null}],
+ {ok, VAcc1} = view_cb({meta, Meta}, VAcc0),
+ DocOpts = case Args#mrargs.conflicts of
+ true -> [conflicts | Args#mrargs.doc_options];
+ _ -> Args#mrargs.doc_options
+ end ++ [{user_ctx, UserCtx}],
+ IncludeDocs = Args#mrargs.include_docs,
+ VAcc2 = lists:foldl(fun(DocId, Acc) ->
+ OpenOpts = [deleted | DocOpts],
+ Row0 = case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ #view_row{key = DocId};
+ {ok, #doc{deleted = true, revs = Revs}} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})},
+ {deleted, true}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ null
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ };
+ {ok, #doc{revs = Revs} = Doc0} ->
+ {RevPos, [RevId | _]} = Revs,
+ Value = {[
+ {rev, couch_doc:rev_to_str({RevPos, RevId})}
+ ]},
+ DocValue = if not IncludeDocs -> undefined; true ->
+ couch_doc:to_json_obj(Doc0, DocOpts)
+ end,
+ #view_row{
+ key = DocId,
+ id = DocId,
+ value = Value,
+ doc = DocValue
+ }
+ end,
+ Row1 = fabric_view:transform_row(Row0),
+ {ok, NewAcc} = view_cb(Row1, Acc),
+ NewAcc
+ end, VAcc1, Keys).
+
+
+all_docs_view_opts(Args, UserCtx) ->
StartKey = case Args#mrargs.start_key of
undefined -> Args#mrargs.start_key_docid;
SKey -> SKey
@@ -925,7 +944,7 @@ all_docs_view_opts(Args, Req) ->
{_, _} -> [{end_key, EndKey}]
end,
[
- {user_ctx, Req#httpd.user_ctx},
+ {user_ctx, UserCtx},
{dir, Args#mrargs.direction},
{start_key, StartKey},
{limit, Args#mrargs.limit},
diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl
index b88cfcd..b9dbb71 100644
--- a/src/couch_views/src/couch_views_util.erl
+++ b/src/couch_views/src/couch_views_util.erl
@@ -14,7 +14,8 @@
-export([
- ddoc_to_mrst/2
+ ddoc_to_mrst/2,
+ validate_args/1
]).
@@ -82,3 +83,145 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}.
+
+
+% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't
+% update start / end keys and also throws a not_implemented error for reduce
+%
+validate_args(#mrargs{} = Args) ->
+ GroupLevel = determine_group_level(Args),
+ Reduce = Args#mrargs.reduce,
+
+ case Reduce =/= undefined orelse Args#mrargs.view_type == red of
+ true -> throw(not_implemented);
+ false -> ok
+ end,
+
+ case Reduce == undefined orelse is_boolean(Reduce) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid `reduce` value.">>)
+ end,
+
+ case {Args#mrargs.view_type, Reduce} of
+ {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of
+ {red, exact, _} -> ok;
+ {red, _, KeyList} when is_list(KeyList) ->
+ Msg = <<"Multi-key fetchs for reduce views must use `group=true`">>,
+ mrverror(Msg);
+ _ -> ok
+ end,
+
+ case Args#mrargs.keys of
+ Keys when is_list(Keys) -> ok;
+ undefined -> ok;
+ _ -> mrverror(<<"`keys` must be an array of strings.">>)
+ end,
+
+ case {Args#mrargs.keys, Args#mrargs.start_key,
+ Args#mrargs.end_key} of
+ {undefined, _, _} -> ok;
+ {[], _, _} -> ok;
+ {[_|_], undefined, undefined} -> ok;
+ _ -> mrverror(<<"`keys` is incompatible with `key`"
+ ", `start_key` and `end_key`">>)
+ end,
+
+ case Args#mrargs.start_key_docid of
+ undefined -> ok;
+ SKDocId0 when is_binary(SKDocId0) -> ok;
+ _ -> mrverror(<<"`start_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.end_key_docid of
+ undefined -> ok;
+ EKDocId0 when is_binary(EKDocId0) -> ok;
+ _ -> mrverror(<<"`end_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.direction of
+ fwd -> ok;
+ rev -> ok;
+ _ -> mrverror(<<"Invalid direction.">>)
+ end,
+
+ case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of
+ {true, _} -> ok;
+ {_, true} -> ok;
+ _ -> mrverror(<<"`limit` must be a positive integer.">>)
+ end,
+
+ case Args#mrargs.skip < 0 of
+ true -> mrverror(<<"`skip` must be >= 0">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel} of
+ {red, exact} -> ok;
+ {_, 0} -> ok;
+ {red, Int} when is_integer(Int), Int >= 0 -> ok;
+ {red, _} -> mrverror(<<"`group_level` must be >= 0">>);
+ {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>)
+ end,
+
+ case Args#mrargs.stable of
+ true -> ok;
+ false -> ok;
+ _ -> mrverror(<<"Invalid value for `stable`.">>)
+ end,
+
+ case Args#mrargs.update of
+ true -> ok;
+ false -> ok;
+ lazy -> ok;
+ _ -> mrverror(<<"Invalid value for `update`.">>)
+ end,
+
+ case is_boolean(Args#mrargs.inclusive_end) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `inclusive_end`.">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.include_docs} of
+ {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>);
+ {_, ID} when is_boolean(ID) -> ok;
+ _ -> mrverror(<<"Invalid value for `include_docs`">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.conflicts} of
+ {_, undefined} -> ok;
+ {map, V} when is_boolean(V) -> ok;
+ {red, undefined} -> ok;
+ {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>);
+ {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>)
+ end,
+
+ case is_boolean(Args#mrargs.sorted) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `sorted`.">>)
+ end,
+
+ Args#mrargs{group_level=GroupLevel}.
+
+
+determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 ->
+ mrverror(<<"Can't specify group=false and group_level>0 at the same time">>);
+
+determine_group_level(#mrargs{group=true, group_level=undefined}) ->
+ exact;
+
+determine_group_level(#mrargs{group_level=GroupLevel}) ->
+ GroupLevel.
+
+
+mrverror(Mesg) ->
+ throw({query_parse_error, Mesg}).
diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs
index 363972b..70dd6e8 100644
--- a/test/elixir/test/basics_test.exs
+++ b/test/elixir/test/basics_test.exs
@@ -316,4 +316,44 @@ defmodule BasicsTest do
# TODO
assert true
end
+
+ @tag :with_db
+ test "_all_docs/queries works", context do
+ db_name = context[:db_name]
+
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: %{:queries => []})
+ assert resp.status_code == 200
+ assert resp.body["results"] == []
+
+ assert Couch.put("/#{db_name}/doc1", body: %{:a => 1}).body["ok"]
+
+ body = %{
+ :queries => [
+ %{:limit => 1},
+ %{:limit => 0}
+ ]
+ }
+ resp = Couch.post("/#{db_name}/_all_docs/queries", body: body)
+ assert resp.status_code == 200
+
+ assert Map.has_key?(resp.body, "results")
+ results = Enum.sort(resp.body["results"])
+ assert length(results) == 2
+ [res1, res2] = results
+
+ assert res1 == %{"offset" => :null, "rows" => [], "total_rows" => 1}
+
+ assert res2["offset"] == :null
+ assert res2["total_rows"] == 1
+ rows = res2["rows"]
+
+ assert length(rows) == 1
+ [row] = rows
+ assert row["id"] == "doc1"
+ assert row["key"] == "doc1"
+
+ val = row["value"]
+ assert Map.has_key?(val, "rev")
+ end
+
end