You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2021/04/16 21:45:22 UTC
[couchdb] 17/24: Move utilities and records from couch_mrview and
couch_index to couch_views
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5ec21191ad2956d393132e6fc3f11711e62d8d2c
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Wed Apr 14 02:12:40 2021 -0400
Move utilities and records from couch_mrview and couch_index to couch_views
* `couch_mrview_util` functions ended up mostly in `couch_views_util`.
* `couch_mrview` validatation functions ended up in `couch_views_validate`
module.
* `couch_mrview_http` functions moved to `couch_views_http_util`. The reason
they didn't end up in `couch_views_http` is because a lot of the functions
there have the exact same names as the ones in `couch_views_http_util`.
There is quite a bit of duplication involved but that is left for another
refactoring in the future. The general flow of control goes from chttpd ->
couch_views_http -> couch_views_http_util.
Most of the changes are just copy and paste with the exception of the
`ddoc_to_mrst/2` function. Previously, there were two almost identical copies
-- one in `couch_mrview_util` and another in `couch_views_util`. Both were used
by different parts of the code. The difference was the couch_views one
optionally disabled reduce functions, and replaced their body with the
`disabled` atom, while the one in `couch_mrview` didn't. Trying to unify them
such that only the `couch_views` one is used, resulted in the inability to
write design documents on server which have custom reduce disabled. That may be
a better behavior, however that should be updated in a separate PR and possibly
a mailing list discussion. So in order to preserve the exisiting behavior,
couch_eval was update to not fail in `try_compile` when design documents are
disabled.
Patches to the rest of the code to update the include path and use the new
utility functions will be updated in a separate commit.
---
src/couch_eval/src/couch_eval.erl | 6 +-
src/couch_views/include/couch_views.hrl | 94 ++++++
src/couch_views/src/couch_views_http_util.erl | 358 ++++++++++++++++++++
src/couch_views/src/couch_views_util.erl | 105 +++++-
src/couch_views/src/couch_views_validate.erl | 460 ++++++++++++++++++++++++++
5 files changed, 1018 insertions(+), 5 deletions(-)
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl
index a6e5965..f87ba97 100644
--- a/src/couch_eval/src/couch_eval.erl
+++ b/src/couch_eval/src/couch_eval.erl
@@ -37,7 +37,7 @@
-type result() :: {doc_id(), [[{any(), any()}]]}.
-type api_mod() :: atom().
-type context() :: {api_mod(), any()}.
--type function_type() :: binary().
+-type function_type() :: binary() | atom().
-type function_name() :: binary().
-type function_src() :: binary().
-type error(_Error) :: no_return().
@@ -117,6 +117,10 @@ with_context(#{language := Language}, Fun) ->
-spec try_compile(context(), function_type(), function_name(), function_src()) -> ok.
+try_compile({_ApiMod, _Ctx}, reduce, <<_/binary>>, disabled) ->
+ % Reduce functions may be disabled. Accept that as a valid configuration.
+ ok;
+
try_compile({ApiMod, Ctx}, FuncType, FuncName, FuncSrc) ->
ApiMod:try_compile(Ctx, FuncType, FuncName, FuncSrc).
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index e28fa74..86f73a3 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -45,3 +45,97 @@
% be used. Use `null` so it can can be round-tripped through json serialization
% with couch_jobs.
-define(VIEW_CURRENT_VSN, null).
+
+
+-record(mrst, {
+ sig=nil,
+ fd=nil,
+ fd_monitor,
+ db_name,
+ idx_name,
+ language,
+ design_opts=[],
+ partitioned=false,
+ lib,
+ views,
+ id_btree=nil,
+ update_seq=0,
+ purge_seq=0,
+ first_build,
+ partial_resp_pid,
+ doc_acc,
+ doc_queue,
+ write_queue,
+ qserver=nil
+}).
+
+
+-record(mrview, {
+ id_num,
+ update_seq=0,
+ purge_seq=0,
+ map_names=[],
+ reduce_funs=[],
+ def,
+ btree=nil,
+ options=[]
+}).
+
+
+-define(MAX_VIEW_LIMIT, 16#10000000).
+
+-record(mrargs, {
+ view_type,
+ reduce,
+
+ preflight_fun,
+
+ start_key,
+ start_key_docid,
+ end_key,
+ end_key_docid,
+ keys,
+
+ direction = fwd,
+ limit = ?MAX_VIEW_LIMIT,
+ skip = 0,
+ group_level = 0,
+ group = undefined,
+ stable = false,
+ update = true,
+ multi_get = false,
+ inclusive_end = true,
+ include_docs = false,
+ doc_options = [],
+ update_seq=false,
+ conflicts,
+ callback,
+ sorted = true,
+ extra = [],
+ page_size = undefined,
+ bookmark=nil
+}).
+
+-record(vacc, {
+ db,
+ req,
+ resp,
+ prepend,
+ etag,
+ should_close = false,
+ buffer = [],
+ bufsize = 0,
+ threshold = 1490,
+ row_sent = false,
+ meta_sent = false,
+ paginated = false,
+ meta = #{}
+}).
+
+
+-record(view_row, {
+ key,
+ id,
+ value,
+ doc
+}).
diff --git a/src/couch_views/src/couch_views_http_util.erl b/src/couch_views/src/couch_views_http_util.erl
new file mode 100644
index 0000000..7af0726
--- /dev/null
+++ b/src/couch_views/src/couch_views_http_util.erl
@@ -0,0 +1,358 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% The reason this module and couch_views_http exist is because they have
+% functions which are named the same but do slightly different things. The
+% general pattern is chttpd code would call into couch_view_http and those
+% function will in turn call into this module.
+
+-module(couch_views_http_util).
+
+-export([
+ prepend_val/1,
+ parse_body_and_query/2,
+ parse_body_and_query/3,
+ parse_params/2,
+ parse_params/3,
+ parse_params/4,
+ view_cb/2,
+ row_to_obj/1,
+ row_to_obj/2,
+ row_to_json/1,
+ row_to_json/2
+]).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
+
+%% these clauses start (and possibly end) the response
+view_cb({error, Reason}, #vacc{resp=undefined}=Acc) ->
+ {ok, Resp} = chttpd:send_error(Acc#vacc.req, Reason),
+ {ok, Acc#vacc{resp=Resp}};
+
+view_cb(complete, #vacc{resp=undefined}=Acc) ->
+ % Nothing in view
+ {ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
+ {ok, Acc#vacc{resp=Resp}};
+
+view_cb(Msg, #vacc{resp=undefined}=Acc) ->
+ %% Start response
+ Headers = [],
+ {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ view_cb(Msg, Acc#vacc{resp=Resp, should_close=true});
+
+%% ---------------------------------------------------
+
+%% From here on down, the response has been started.
+
+view_cb({error, Reason}, #vacc{resp=Resp}=Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_error(Resp, Reason),
+ {ok, Acc#vacc{resp=Resp1}};
+
+view_cb(complete, #vacc{resp=Resp, buffer=Buf, threshold=Max}=Acc) ->
+ % Finish view output and possibly end the response
+ {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, "\r\n]}", Max),
+ case Acc#vacc.should_close of
+ true ->
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, Acc#vacc{resp=Resp2}};
+ _ ->
+ {ok, Acc#vacc{resp=Resp1, meta_sent=false, row_sent=false,
+ prepend=",\r\n", buffer=[], bufsize=0}}
+ end;
+
+view_cb({meta, Meta}, #vacc{meta_sent=false, row_sent=false}=Acc) ->
+ % Sending metadata as we've not sent it or any row yet
+ Parts = case couch_util:get_value(total, Meta) of
+ undefined -> [];
+ Total -> [io_lib:format("\"total_rows\":~p", [Total])]
+ end ++ case couch_util:get_value(offset, Meta) of
+ undefined -> [];
+ Offset -> [io_lib:format("\"offset\":~p", [Offset])]
+ end ++ case couch_util:get_value(update_seq, Meta) of
+ undefined -> [];
+ null ->
+ ["\"update_seq\":null"];
+ UpdateSeq when is_integer(UpdateSeq) ->
+ [io_lib:format("\"update_seq\":~B", [UpdateSeq])];
+ UpdateSeq when is_binary(UpdateSeq) ->
+ [io_lib:format("\"update_seq\":\"~s\"", [UpdateSeq])]
+ end ++ ["\"rows\":["],
+ Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"],
+ {ok, AccOut} = maybe_flush_response(Acc, Chunk, iolist_size(Chunk)),
+ {ok, AccOut#vacc{prepend="", meta_sent=true}};
+
+view_cb({meta, _Meta}, #vacc{}=Acc) ->
+ %% ignore metadata
+ {ok, Acc};
+
+view_cb({row, Row}, #vacc{meta_sent=false}=Acc) ->
+ %% sorted=false and row arrived before meta
+ % Adding another row
+ Chunk = [prepend_val(Acc), "{\"rows\":[\r\n", row_to_json(Row)],
+ maybe_flush_response(Acc#vacc{meta_sent=true, row_sent=true}, Chunk, iolist_size(Chunk));
+
+view_cb({row, Row}, #vacc{meta_sent=true}=Acc) ->
+ % Adding another row
+ Chunk = [prepend_val(Acc), row_to_json(Row)],
+ maybe_flush_response(Acc#vacc{row_sent=true}, Chunk, iolist_size(Chunk)).
+
+
+maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len)
+ when Size > 0 andalso (Size + Len) > Max ->
+ #vacc{buffer = Buffer, resp = Resp} = Acc,
+ {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
+ {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}};
+maybe_flush_response(Acc0, Data, Len) ->
+ #vacc{buffer = Buf, bufsize = Size} = Acc0,
+ Acc = Acc0#vacc{
+ prepend = ",\r\n",
+ buffer = [Buf | Data],
+ bufsize = Size + Len
+ },
+ {ok, Acc}.
+
+prepend_val(#vacc{prepend=Prepend}) ->
+ case Prepend of
+ undefined ->
+ "";
+ _ ->
+ Prepend
+ end.
+
+
+row_to_json(Row) ->
+ ?JSON_ENCODE(row_to_obj(Row)).
+
+
+row_to_json(Kind, Row) ->
+ ?JSON_ENCODE(row_to_obj(Kind, Row)).
+
+
+row_to_obj(Row) ->
+ Id = couch_util:get_value(id, Row),
+ row_to_obj(Id, Row).
+
+
+row_to_obj(error, Row) ->
+ % Special case for _all_docs request with KEYS to
+ % match prior behavior.
+ Key = couch_util:get_value(key, Row),
+ Val = couch_util:get_value(value, Row),
+ Reason = couch_util:get_value(reason, Row),
+ ReasonProp = if Reason == undefined -> []; true ->
+ [{reason, Reason}]
+ end,
+ {[{key, Key}, {error, Val}] ++ ReasonProp};
+row_to_obj(Id0, Row) ->
+ Id = case Id0 of
+ undefined -> [];
+ Id0 -> [{id, Id0}]
+ end,
+ Key = couch_util:get_value(key, Row, null),
+ Val = couch_util:get_value(value, Row),
+ Doc = case couch_util:get_value(doc, Row) of
+ undefined -> [];
+ Doc0 -> [{doc, Doc0}]
+ end,
+ {Id ++ [{key, Key}, {value, Val}] ++ Doc}.
+
+
+parse_params(#httpd{}=Req, Keys) ->
+ parse_params(chttpd:qs(Req), Keys);
+parse_params(Props, Keys) ->
+ Args = #mrargs{},
+ parse_params(Props, Keys, Args).
+
+
+parse_params(Props, Keys, Args) ->
+ parse_params(Props, Keys, Args, []).
+
+parse_params(Props, Keys, #mrargs{}=Args0, Options) ->
+ IsDecoded = lists:member(decoded, Options),
+ Args1 = case lists:member(keep_group_level, Options) of
+ true ->
+ Args0;
+ _ ->
+ % group_level set to undefined to detect if explicitly set by user
+ Args0#mrargs{keys=Keys, group=undefined, group_level=undefined}
+ end,
+ lists:foldl(fun({K, V}, Acc) ->
+ parse_param(K, V, Acc, IsDecoded)
+ end, Args1, Props).
+
+
+parse_body_and_query(#httpd{method='POST'} = Req, Keys) ->
+ Props = chttpd:json_body_obj(Req),
+ parse_body_and_query(Req, Props, Keys);
+
+parse_body_and_query(Req, Keys) ->
+ parse_params(chttpd:qs(Req), Keys, #mrargs{keys=Keys, group=undefined,
+ group_level=undefined}, [keep_group_level]).
+
+parse_body_and_query(Req, {Props}, Keys) ->
+ Args = #mrargs{keys=Keys, group=undefined, group_level=undefined},
+ BodyArgs = parse_params(Props, Keys, Args, [decoded]),
+ parse_params(chttpd:qs(Req), Keys, BodyArgs, [keep_group_level]).
+
+parse_param(Key, Val, Args, IsDecoded) when is_binary(Key) ->
+ parse_param(binary_to_list(Key), Val, Args, IsDecoded);
+parse_param(Key, Val, Args, IsDecoded) ->
+ case Key of
+ "" ->
+ Args;
+ "reduce" ->
+ Args#mrargs{reduce=parse_boolean(Val)};
+ "key" when IsDecoded ->
+ Args#mrargs{start_key=Val, end_key=Val};
+ "key" ->
+ JsonKey = ?JSON_DECODE(Val),
+ Args#mrargs{start_key=JsonKey, end_key=JsonKey};
+ "keys" when IsDecoded ->
+ Args#mrargs{keys=Val};
+ "keys" ->
+ Args#mrargs{keys=?JSON_DECODE(Val)};
+ "startkey" when IsDecoded ->
+ Args#mrargs{start_key=Val};
+ "start_key" when IsDecoded ->
+ Args#mrargs{start_key=Val};
+ "startkey" ->
+ Args#mrargs{start_key=?JSON_DECODE(Val)};
+ "start_key" ->
+ Args#mrargs{start_key=?JSON_DECODE(Val)};
+ "startkey_docid" ->
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
+ "start_key_doc_id" ->
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
+ "endkey" when IsDecoded ->
+ Args#mrargs{end_key=Val};
+ "end_key" when IsDecoded ->
+ Args#mrargs{end_key=Val};
+ "endkey" ->
+ Args#mrargs{end_key=?JSON_DECODE(Val)};
+ "end_key" ->
+ Args#mrargs{end_key=?JSON_DECODE(Val)};
+ "endkey_docid" ->
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
+ "end_key_doc_id" ->
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
+ "limit" ->
+ Args#mrargs{limit=parse_pos_int(Val)};
+ "page_size" ->
+ Args#mrargs{page_size=parse_pos_int(Val)};
+ "stale" when Val == "ok" orelse Val == <<"ok">> ->
+ Args#mrargs{stable=true, update=false};
+ "stale" when Val == "update_after" orelse Val == <<"update_after">> ->
+ Args#mrargs{stable=true, update=lazy};
+ "stale" ->
+ throw({query_parse_error, <<"Invalid value for `stale`.">>});
+ "stable" when Val == "true" orelse Val == <<"true">> orelse Val == true ->
+ Args#mrargs{stable=true};
+ "stable" when Val == "false" orelse Val == <<"false">> orelse Val == false ->
+ Args#mrargs{stable=false};
+ "stable" ->
+ throw({query_parse_error, <<"Invalid value for `stable`.">>});
+ "update" when Val == "true" orelse Val == <<"true">> orelse Val == true ->
+ Args#mrargs{update=true};
+ "update" when Val == "false" orelse Val == <<"false">> orelse Val == false ->
+ Args#mrargs{update=false};
+ "update" when Val == "lazy" orelse Val == <<"lazy">> ->
+ Args#mrargs{update=lazy};
+ "update" ->
+ throw({query_parse_error, <<"Invalid value for `update`.">>});
+ "descending" ->
+ case parse_boolean(Val) of
+ true -> Args#mrargs{direction=rev};
+ _ -> Args#mrargs{direction=fwd}
+ end;
+ "skip" ->
+ Args#mrargs{skip=parse_pos_int(Val)};
+ "group" ->
+ Args#mrargs{group=parse_boolean(Val)};
+ "group_level" ->
+ Args#mrargs{group_level=parse_pos_int(Val)};
+ "inclusive_end" ->
+ Args#mrargs{inclusive_end=parse_boolean(Val)};
+ "include_docs" ->
+ Args#mrargs{include_docs=parse_boolean(Val)};
+ "attachments" ->
+ case parse_boolean(Val) of
+ true ->
+ Opts = Args#mrargs.doc_options,
+ Args#mrargs{doc_options=[attachments|Opts]};
+ false ->
+ Args
+ end;
+ "att_encoding_info" ->
+ case parse_boolean(Val) of
+ true ->
+ Opts = Args#mrargs.doc_options,
+ Args#mrargs{doc_options=[att_encoding_info|Opts]};
+ false ->
+ Args
+ end;
+ "update_seq" ->
+ Args#mrargs{update_seq=parse_boolean(Val)};
+ "conflicts" ->
+ Args#mrargs{conflicts=parse_boolean(Val)};
+ "callback" ->
+ Args#mrargs{callback=couch_util:to_binary(Val)};
+ "sorted" ->
+ Args#mrargs{sorted=parse_boolean(Val)};
+ "partition" ->
+ Partition = couch_util:to_binary(Val),
+ couch_partition:validate_partition(Partition),
+ couch_views_util:set_extra(Args, partition, Partition);
+ _ ->
+ BKey = couch_util:to_binary(Key),
+ BVal = couch_util:to_binary(Val),
+ Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]}
+ end.
+
+
+parse_boolean(true) ->
+ true;
+parse_boolean(false) ->
+ false;
+
+parse_boolean(Val) when is_binary(Val) ->
+ parse_boolean(?b2l(Val));
+
+parse_boolean(Val) ->
+ case string:to_lower(Val) of
+ "true" -> true;
+ "false" -> false;
+ _ ->
+ Msg = io_lib:format("Invalid boolean parameter: ~p", [Val]),
+ throw({query_parse_error, ?l2b(Msg)})
+ end.
+
+parse_int(Val) when is_integer(Val) ->
+ Val;
+parse_int(Val) ->
+ case (catch list_to_integer(Val)) of
+ IntVal when is_integer(IntVal) ->
+ IntVal;
+ _ ->
+ Msg = io_lib:format("Invalid value for integer: ~p", [Val]),
+ throw({query_parse_error, ?l2b(Msg)})
+ end.
+
+parse_pos_int(Val) ->
+ case parse_int(Val) of
+ IntVal when IntVal >= 0 ->
+ IntVal;
+ _ ->
+ Fmt = "Invalid value for positive integer: ~p",
+ Msg = io_lib:format(Fmt, [Val]),
+ throw({query_parse_error, ?l2b(Msg)})
+ end.
diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl
index 7040020..287d4ba 100644
--- a/src/couch_views/src/couch_views_util.erl
+++ b/src/couch_views/src/couch_views_util.erl
@@ -19,12 +19,16 @@
validate_args/1,
validate_args/2,
is_paginated/1,
- active_tasks_info/5
+ active_tasks_info/5,
+ set_view_type/3,
+ set_extra/3,
+ get_view_queries/1,
+ get_view_keys/1,
+ extract_view/4
]).
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_mrview/include/couch_mrview.hrl").
-include("couch_views.hrl").
@@ -80,10 +84,53 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
design_opts=DesignOpts,
partitioned=Partitioned
},
- SigInfo = {Views1, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
+ SigInfo = {Views1, Language, DesignOpts, sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}.
+set_view_type(_Args, _ViewName, []) ->
+ throw({not_found, missing_named_view});
+
+set_view_type(Args, ViewName, [View | Rest]) ->
+ RedNames = [N || {N, _} <- View#mrview.reduce_funs],
+ case lists:member(ViewName, RedNames) of
+ true ->
+ case Args#mrargs.reduce of
+ false -> Args#mrargs{view_type=map};
+ _ -> Args#mrargs{view_type=red}
+ end;
+ false ->
+ case lists:member(ViewName, View#mrview.map_names) of
+ true -> Args#mrargs{view_type=map};
+ false -> set_view_type(Args, ViewName, Rest)
+ end
+ end.
+
+
+set_extra(#mrargs{} = Args, Key, Value) ->
+ Extra0 = Args#mrargs.extra,
+ Extra1 = lists:ukeysort(1, [{Key, Value} | Extra0]),
+ Args#mrargs{extra = Extra1}.
+
+
+extract_view(_Lang, _Args, _ViewName, []) ->
+ throw({not_found, missing_named_view});
+
+extract_view(Lang, #mrargs{view_type=map}=Args, Name, [View | Rest]) ->
+ Names = View#mrview.map_names ++ [N || {N, _} <- View#mrview.reduce_funs],
+ case lists:member(Name, Names) of
+ true -> {map, View, Args};
+ _ -> extract_view(Lang, Args, Name, Rest)
+ end;
+
+extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) ->
+ RedNames = [N || {N, _} <- View#mrview.reduce_funs],
+ case lists:member(Name, RedNames) of
+ true -> {red, {index_of(Name, RedNames), Lang, View}, Args};
+ false -> extract_view(Lang, Args, Name, Rest)
+ end.
+
+
collate_fun(View) ->
#mrview{
options = Options
@@ -122,7 +169,7 @@ validate_args(Args) ->
validate_args(Args, []).
-% This is mostly a copy of couch_mrview_util:validate_args/1 but it doesn't
+% This is mostly a copy of couch_validate:validate_args/1 but it doesn't
% update start / end keys and also throws a not_implemented error for reduce
%
validate_args(#mrargs{} = Args, Opts) ->
@@ -366,3 +413,53 @@ convert_seq_to_stamp(Seq) ->
VS = integer_to_list(Stamp) ++ "-" ++ integer_to_list(Batch) ++ "-"
++ integer_to_list(DocNumber),
list_to_binary(VS).
+
+
+get_view_queries({Props}) ->
+ case couch_util:get_value(<<"queries">>, Props) of
+ undefined ->
+ undefined;
+ Queries when is_list(Queries) ->
+ Queries;
+ _ ->
+ throw({bad_request, "`queries` member must be an array."})
+ end.
+
+
+get_view_keys({Props}) ->
+ case couch_util:get_value(<<"keys">>, Props) of
+ undefined ->
+ undefined;
+ Keys when is_list(Keys) ->
+ Keys;
+ _ ->
+ throw({bad_request, "`keys` member must be an array."})
+ end.
+
+
+sort_lib({Lib}) ->
+ sort_lib(Lib, []).
+
+sort_lib([], LAcc) ->
+ lists:keysort(1, LAcc);
+
+sort_lib([{LName, {LObj}}|Rest], LAcc) ->
+ LSorted = sort_lib(LObj, []), % descend into nested object
+ sort_lib(Rest, [{LName, LSorted}|LAcc]);
+
+sort_lib([{LName, LCode}|Rest], LAcc) ->
+ sort_lib(Rest, [{LName, LCode}|LAcc]).
+
+
+index_of(Key, List) ->
+ index_of(Key, List, 1).
+
+
+index_of(_, [], _) ->
+ throw({error, missing_named_view});
+
+index_of(Key, [Key | _], Idx) ->
+ Idx;
+
+index_of(Key, [_ | Rest], Idx) ->
+ index_of(Key, Rest, Idx+1).
diff --git a/src/couch_views/src/couch_views_validate.erl b/src/couch_views/src/couch_views_validate.erl
new file mode 100644
index 0000000..558f65d
--- /dev/null
+++ b/src/couch_views/src/couch_views_validate.erl
@@ -0,0 +1,460 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_validate).
+
+
+-export([
+ validate_args/1,
+ validate_args/3,
+ validate_ddoc/2
+]).
+
+
+-define(LOWEST_KEY, null).
+-define(HIGHEST_KEY, {<<255, 255, 255, 255>>}).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
+
+
+% There is another almost identical validate_args in couch_views_util. They
+% should probably be merged at some point in the future.
+%
+validate_args(Args) ->
+ GroupLevel = determine_group_level(Args),
+ Reduce = Args#mrargs.reduce,
+ case Reduce == undefined orelse is_boolean(Reduce) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid `reduce` value.">>)
+ end,
+
+ case {Args#mrargs.view_type, Reduce} of
+ {map, true} -> mrverror(<<"Reduce is invalid for map-only views.">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel, Args#mrargs.keys} of
+ {red, exact, _} -> ok;
+ {red, _, KeyList} when is_list(KeyList) ->
+ Msg = <<"Multi-key fetches for reduce views must use `group=true`">>,
+ mrverror(Msg);
+ _ -> ok
+ end,
+
+ case Args#mrargs.keys of
+ Keys when is_list(Keys) -> ok;
+ undefined -> ok;
+ _ -> mrverror(<<"`keys` must be an array of strings.">>)
+ end,
+
+ case {Args#mrargs.keys, Args#mrargs.start_key,
+ Args#mrargs.end_key} of
+ {undefined, _, _} -> ok;
+ {[], _, _} -> ok;
+ {[_|_], undefined, undefined} -> ok;
+ _ -> mrverror(<<"`keys` is incompatible with `key`"
+ ", `start_key` and `end_key`">>)
+ end,
+
+ case Args#mrargs.start_key_docid of
+ undefined -> ok;
+ SKDocId0 when is_binary(SKDocId0) -> ok;
+ _ -> mrverror(<<"`start_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.end_key_docid of
+ undefined -> ok;
+ EKDocId0 when is_binary(EKDocId0) -> ok;
+ _ -> mrverror(<<"`end_key_docid` must be a string.">>)
+ end,
+
+ case Args#mrargs.direction of
+ fwd -> ok;
+ rev -> ok;
+ _ -> mrverror(<<"Invalid direction.">>)
+ end,
+
+ case {Args#mrargs.limit >= 0, Args#mrargs.limit == undefined} of
+ {true, _} -> ok;
+ {_, true} -> ok;
+ _ -> mrverror(<<"`limit` must be a positive integer.">>)
+ end,
+
+ case Args#mrargs.skip < 0 of
+ true -> mrverror(<<"`skip` must be >= 0">>);
+ _ -> ok
+ end,
+
+ case {Args#mrargs.view_type, GroupLevel} of
+ {red, exact} -> ok;
+ {_, 0} -> ok;
+ {red, Int} when is_integer(Int), Int >= 0 -> ok;
+ {red, _} -> mrverror(<<"`group_level` must be >= 0">>);
+ {map, _} -> mrverror(<<"Invalid use of grouping on a map view.">>)
+ end,
+
+ case Args#mrargs.stable of
+ true -> ok;
+ false -> ok;
+ _ -> mrverror(<<"Invalid value for `stable`.">>)
+ end,
+
+ case Args#mrargs.update of
+ true -> ok;
+ false -> ok;
+ lazy -> ok;
+ _ -> mrverror(<<"Invalid value for `update`.">>)
+ end,
+
+ case is_boolean(Args#mrargs.inclusive_end) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `inclusive_end`.">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.include_docs} of
+ {red, true} -> mrverror(<<"`include_docs` is invalid for reduce">>);
+ {_, ID} when is_boolean(ID) -> ok;
+ _ -> mrverror(<<"Invalid value for `include_docs`">>)
+ end,
+
+ case {Args#mrargs.view_type, Args#mrargs.conflicts} of
+ {_, undefined} -> ok;
+ {map, V} when is_boolean(V) -> ok;
+ {red, undefined} -> ok;
+ {map, _} -> mrverror(<<"Invalid value for `conflicts`.">>);
+ {red, _} -> mrverror(<<"`conflicts` is invalid for reduce views.">>)
+ end,
+
+ SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of
+ {fwd, undefined} -> <<>>;
+ {rev, undefined} -> <<255>>;
+ {_, SKDocId1} -> SKDocId1
+ end,
+
+ EKDocId = case {Args#mrargs.direction, Args#mrargs.end_key_docid} of
+ {fwd, undefined} -> <<255>>;
+ {rev, undefined} -> <<>>;
+ {_, EKDocId1} -> EKDocId1
+ end,
+
+ case is_boolean(Args#mrargs.sorted) of
+ true -> ok;
+ _ -> mrverror(<<"Invalid value for `sorted`.">>)
+ end,
+
+ Args#mrargs{
+ start_key_docid=SKDocId,
+ end_key_docid=EKDocId,
+ group_level=GroupLevel
+ }.
+
+
+validate_args(Db, DDoc, Args0) ->
+ {ok, State} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ Args1 = apply_limit(State#mrst.partitioned, Args0),
+ validate_args(State, Args1).
+
+
+validate_ddoc(#{} = Db, DDoc) ->
+ DbName = fabric2_db:name(Db),
+ IsPartitioned = fabric2_db:is_partitioned(Db),
+ validate_ddoc(DbName, IsPartitioned, DDoc).
+
+
+% Private functions
+
+validate_ddoc(DbName, _IsDbPartitioned, DDoc) ->
+ ok = validate_ddoc_fields(DDoc#doc.body),
+ GetName = fun
+ (#mrview{map_names = [Name | _]}) -> Name;
+ (#mrview{reduce_funs = [{Name, _} | _]}) -> Name;
+ (_) -> null
+ end,
+ ValidateView = fun(Ctx, #mrview{def=MapSrc, reduce_funs=Reds}=View) ->
+ couch_eval:try_compile(Ctx, map, GetName(View), MapSrc),
+ lists:foreach(fun
+ ({_RedName, <<"_sum", _/binary>>}) ->
+ ok;
+ ({_RedName, <<"_count", _/binary>>}) ->
+ ok;
+ ({_RedName, <<"_stats", _/binary>>}) ->
+ ok;
+ ({_RedName, <<"_approx_count_distinct", _/binary>>}) ->
+ ok;
+ ({_RedName, <<"_", _/binary>> = Bad}) ->
+ Msg = ["`", Bad, "` is not a supported reduce function."],
+ throw({invalid_design_doc, Msg});
+ ({RedName, RedSrc}) ->
+ couch_eval:try_compile(Ctx, reduce, RedName, RedSrc)
+ end, Reds)
+ end,
+ {ok, #mrst{
+ language = Lang,
+ views = Views
+ }} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+
+ Views =/= [] andalso couch_eval:with_context(#{language => Lang}, fun (Ctx) ->
+ lists:foreach(fun(V) -> ValidateView(Ctx, V) end, Views)
+ end),
+ ok.
+
+
+validate_args(#mrst{} = State, Args0) ->
+ Args = validate_args(Args0),
+
+ ViewPartitioned = State#mrst.partitioned,
+ Partition = get_extra(Args, partition),
+
+ case {ViewPartitioned, Partition} of
+ {true, undefined} ->
+ Msg1 = <<"`partition` parameter is mandatory "
+ "for queries to this view.">>,
+ mrverror(Msg1);
+ {true, _} ->
+ apply_partition(Args, Partition);
+ {false, undefined} ->
+ Args;
+ {false, Value} when is_binary(Value) ->
+ Msg2 = <<"`partition` parameter is not "
+ "supported in this design doc">>,
+ mrverror(Msg2)
+ end.
+
+
+validate_ddoc_fields(DDoc) ->
+ MapFuncType = map_function_type(DDoc),
+ lists:foreach(fun(Path) ->
+ validate_ddoc_fields(DDoc, Path)
+ end, [
+ [{<<"filters">>, object}, {any, [object, string]}],
+ [{<<"language">>, string}],
+ [{<<"lists">>, object}, {any, [object, string]}],
+ [{<<"options">>, object}],
+ [{<<"options">>, object}, {<<"include_design">>, boolean}],
+ [{<<"options">>, object}, {<<"local_seq">>, boolean}],
+ [{<<"options">>, object}, {<<"partitioned">>, boolean}],
+ [{<<"rewrites">>, [string, array]}],
+ [{<<"shows">>, object}, {any, [object, string]}],
+ [{<<"updates">>, object}, {any, [object, string]}],
+ [{<<"validate_doc_update">>, string}],
+ [{<<"views">>, object}, {<<"lib">>, object}],
+ [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}],
+ [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}]
+ ]),
+ require_map_function_for_views(DDoc),
+ ok.
+
+
+require_map_function_for_views({Props}) ->
+ case couch_util:get_value(<<"views">>, Props) of
+ undefined -> ok;
+ {Views} ->
+ lists:foreach(fun
+ ({<<"lib">>, _}) -> ok;
+ ({Key, {Value}}) ->
+ case couch_util:get_value(<<"map">>, Value) of
+ undefined -> throw({invalid_design_doc,
+ <<"View `", Key/binary, "` must contain map function">>});
+ _ -> ok
+ end
+ end, Views),
+ ok
+ end.
+
+
+validate_ddoc_fields(DDoc, Path) ->
+ case validate_ddoc_fields(DDoc, Path, []) of
+ ok -> ok;
+ {error, {FailedPath0, Type0}} ->
+ FailedPath = iolist_to_binary(join(FailedPath0, <<".">>)),
+ Type = format_type(Type0),
+ throw({invalid_design_doc,
+ <<"`", FailedPath/binary, "` field must have ",
+ Type/binary, " type">>})
+ end.
+
+validate_ddoc_fields(undefined, _, _) ->
+ ok;
+
+validate_ddoc_fields(_, [], _) ->
+ ok;
+
+validate_ddoc_fields({KVS}=Props, [{any, Type} | Rest], Acc) ->
+ lists:foldl(fun
+ ({Key, _}, ok) ->
+ validate_ddoc_fields(Props, [{Key, Type} | Rest], Acc);
+ ({_, _}, {error, _}=Error) ->
+ Error
+ end, ok, KVS);
+
+validate_ddoc_fields({KVS}=Props, [{Key, Type} | Rest], Acc) ->
+ case validate_ddoc_field(Props, {Key, Type}) of
+ ok ->
+ validate_ddoc_fields(couch_util:get_value(Key, KVS),
+ Rest,
+ [Key | Acc]);
+ error ->
+ {error, {[Key | Acc], Type}};
+ {error, Key1} ->
+ {error, {[Key1 | Acc], Type}}
+ end.
+
+
+validate_ddoc_field(undefined, Type) when is_atom(Type) ->
+ ok;
+
+validate_ddoc_field(_, any) ->
+ ok;
+
+validate_ddoc_field(Value, Types) when is_list(Types) ->
+ lists:foldl(fun
+ (_, ok) -> ok;
+ (Type, _) -> validate_ddoc_field(Value, Type)
+ end, error, Types);
+validate_ddoc_field(Value, string) when is_binary(Value) ->
+ ok;
+
+validate_ddoc_field(Value, array) when is_list(Value) ->
+ ok;
+
+validate_ddoc_field({Value}, object) when is_list(Value) ->
+ ok;
+
+validate_ddoc_field(Value, boolean) when is_boolean(Value) ->
+ ok;
+
+validate_ddoc_field({Props}, {any, Type}) ->
+ validate_ddoc_field1(Props, Type);
+
+validate_ddoc_field({Props}, {Key, Type}) ->
+ validate_ddoc_field(couch_util:get_value(Key, Props), Type);
+
+validate_ddoc_field(_, _) ->
+ error.
+
+
+validate_ddoc_field1([], _) ->
+ ok;
+
+validate_ddoc_field1([{Key, Value} | Rest], Type) ->
+ case validate_ddoc_field(Value, Type) of
+ ok ->
+ validate_ddoc_field1(Rest, Type);
+ error ->
+ {error, Key}
+ end.
+
+
+map_function_type({Props}) ->
+ case couch_util:get_value(<<"language">>, Props) of
+ <<"query">> -> object;
+ _ -> string
+ end.
+
+
+format_type(Type) when is_atom(Type) ->
+ ?l2b(atom_to_list(Type));
+
+format_type(Types) when is_list(Types) ->
+ iolist_to_binary(join(lists:map(fun atom_to_list/1, Types), <<" or ">>)).
+
+
+join(L, Sep) ->
+ join(L, Sep, []).
+
+
+join([H|[]], _, Acc) ->
+ [H | Acc];
+
+join([H|T], Sep, Acc) ->
+ join(T, Sep, [Sep, H | Acc]).
+
+
+determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=undefined}) ->
+ 0;
+
+determine_group_level(#mrargs{group=false, group_level=Level}) when Level > 0 ->
+ mrverror(<<"Can't specify group=false and group_level>0 at the same time">>);
+
+determine_group_level(#mrargs{group=true, group_level=undefined}) ->
+ exact;
+
+determine_group_level(#mrargs{group_level=GroupLevel}) ->
+ GroupLevel.
+
+
+mrverror(Mesg) ->
+ throw({query_parse_error, Mesg}).
+
+
+apply_partition(#mrargs{keys=[{p, _, _} | _]} = Args, _Partition) ->
+ Args; % already applied
+
+apply_partition(#mrargs{keys=Keys} = Args, Partition) when Keys /= undefined ->
+ Args#mrargs{keys=[{p, Partition, K} || K <- Keys]};
+
+apply_partition(#mrargs{start_key={p, _, _}, end_key={p, _, _}} = Args, _Partition) ->
+ Args; % already applied.
+
+apply_partition(Args, Partition) ->
+ #mrargs{
+ direction = Dir,
+ start_key = StartKey,
+ end_key = EndKey
+ } = Args,
+
+ {DefSK, DefEK} = case Dir of
+ fwd -> {?LOWEST_KEY, ?HIGHEST_KEY};
+ rev -> {?HIGHEST_KEY, ?LOWEST_KEY}
+ end,
+
+ SK0 = if StartKey /= undefined -> StartKey; true -> DefSK end,
+ EK0 = if EndKey /= undefined -> EndKey; true -> DefEK end,
+
+ Args#mrargs{
+ start_key = {p, Partition, SK0},
+ end_key = {p, Partition, EK0}
+ }.
+
+
+get_extra(#mrargs{} = Args, Key) ->
+ couch_util:get_value(Key, Args#mrargs.extra).
+
+
+apply_limit(ViewPartitioned, Args) ->
+ Options = Args#mrargs.extra,
+ IgnorePQLimit = lists:keyfind(ignore_partition_query_limit, 1, Options),
+ LimitType = case {ViewPartitioned, IgnorePQLimit} of
+ {true, false} -> "partition_query_limit";
+ {true, _} -> "query_limit";
+ {false, _} -> "query_limit"
+ end,
+
+ MaxLimit = config:get_integer("query_server_config",
+ LimitType, ?MAX_VIEW_LIMIT),
+
+ % Set the highest limit possible if a user has not
+ % specified a limit
+ Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of
+ true -> Args#mrargs{limit = MaxLimit};
+ false -> Args
+ end,
+
+ if Args1#mrargs.limit =< MaxLimit -> Args1; true ->
+ Fmt = "Limit is too large, must not exceed ~p",
+ mrverror(io_lib:format(Fmt, [MaxLimit]))
+ end.