You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2019/01/23 06:31:01 UTC
[couchdb] branch master updated: Make _all_docs and view query
limits configurable
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1fdfe46 Make _all_docs and view query limits configurable
1fdfe46 is described below
commit 1fdfe466223380d1ee7c82e21283e161290aed87
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Dec 6 13:03:18 2018 +0200
Make _all_docs and view query limits configurable
This allows us to set a maximun allowed number of documents
to be returned for a global or a partitioned view query and
_all_docs query.
Co-authored-by: Paul J. Davis <pa...@gmail.com>
---
rel/overlay/etc/default.ini | 2 +
src/couch_mrview/include/couch_mrview.hrl | 3 +-
src/couch_mrview/src/couch_mrview_util.erl | 30 +++++++++-
test/elixir/test/partition_all_docs_test.exs | 82 ++++++++++++++++++++++------
test/elixir/test/partition_view_test.exs | 60 ++++++++++++++++++++
5 files changed, 155 insertions(+), 22 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 9a332ac..b9d51af 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -293,6 +293,8 @@ os_process_limit = 100
; Timeout for how long a response from a busy view group server can take.
; "infinity" is also a valid configuration value.
;group_info_timeout = 5000
+;query_limit = 268435456
+;partition_query_limit = 268435456
[mango]
; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index e17aaba..29fe52b 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -60,6 +60,7 @@
view_states=nil
}).
+-define(MAX_VIEW_LIMIT, 16#10000000).
-record(mrargs, {
view_type,
@@ -74,7 +75,7 @@
keys,
direction = fwd,
- limit = 16#10000000,
+ limit = ?MAX_VIEW_LIMIT,
skip = 0,
group_level = 0,
group = undefined,
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index b879d12..eb68124 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -496,9 +496,10 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
-validate_args(Db, DDoc, Args) ->
+validate_args(Db, DDoc, Args0) ->
{ok, State} = couch_mrview_index:init(Db, DDoc),
- validate_args(State, Args).
+ Args1 = apply_limit(State#mrst.partitioned, Args0),
+ validate_args(State, Args1).
validate_args(#mrst{} = State, Args0) ->
@@ -523,6 +524,28 @@ validate_args(#mrst{} = State, Args0) ->
end.
+apply_limit(ViewPartitioned, Args) ->
+ LimitType = case ViewPartitioned of
+ true -> "partition_query_limit";
+ false -> "query_limit"
+ end,
+
+ MaxLimit = config:get_integer("query_server_config",
+ LimitType, ?MAX_VIEW_LIMIT),
+
+ % Set the highest limit possible if a user has not
+ % specified a limit
+ Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of
+ true -> Args#mrargs{limit = MaxLimit};
+ false -> Args
+ end,
+
+ if Args1#mrargs.limit =< MaxLimit -> Args1; true ->
+ Fmt = "Limit is too large, must not exceed ~p",
+ mrverror(io_lib:format(Fmt, [MaxLimit]))
+ end.
+
+
validate_all_docs_args(Db, Args0) ->
Args = validate_args(Args0),
@@ -533,7 +556,8 @@ validate_all_docs_args(Db, Args0) ->
{false, <<_/binary>>} ->
mrverror(<<"`partition` parameter is not supported on this db">>);
{_, <<_/binary>>} ->
- apply_all_docs_partition(Args, Partition);
+ Args1 = apply_limit(true, Args),
+ apply_all_docs_partition(Args1, Partition);
_ ->
Args
end.
diff --git a/test/elixir/test/partition_all_docs_test.exs b/test/elixir/test/partition_all_docs_test.exs
index 308c1b4..87bab34 100644
--- a/test/elixir/test/partition_all_docs_test.exs
+++ b/test/elixir/test/partition_all_docs_test.exs
@@ -116,36 +116,82 @@ defmodule PartitionAllDocsTest do
assert ids == ["foo:22"]
end
+ test "partition all docs can set query limits", context do
+ set_config({"query_server_config", "partition_query_limit", "2000"})
+
+ db_name = context[:db_name]
+ create_partition_docs(db_name)
+ create_partition_ddoc(db_name)
+
+ url = "/#{db_name}/_partition/foo/_all_docs"
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 20
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 20
+
+ resp = Couch.get(url)
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 50
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2000
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 50
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2001
+ }
+ )
+
+ assert resp.status_code == 400
+ %{:body => %{"reason" => reason}} = resp
+ assert Regex.match?(~r/Limit is too large/, reason)
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2000,
+ skip: 25
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 25
+ end
+
# This test is timing based so it could be a little flaky.
# If that turns out to be the case we should probably just skip it
test "partition _all_docs with timeout", context do
- on_exit(fn ->
- resp = Couch.get("/_membership")
- %{:body => body} = resp
-
- Enum.each(body["all_nodes"], fn node ->
- resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"3600000\"")
- assert resp.status_code == 200
- end)
- end)
-
- resp = Couch.get("/_membership")
- %{:body => body} = resp
-
- Enum.each(body["all_nodes"], fn node ->
- resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"1\"")
- assert resp.status_code == 200
- end)
+ set_config({"fabric", "partition_view_timeout", "1"})
db_name = context[:db_name]
create_partition_docs(db_name)
retry_until(fn ->
url = "/#{db_name}/_partition/foo/_all_docs"
+
case Couch.get(url) do
%{:body => %{"reason" => reason}} ->
Regex.match?(~r/not be processed in a reasonable amount of time./, reason)
- _ ->
+
+ _ ->
false
end
end)
diff --git a/test/elixir/test/partition_view_test.exs b/test/elixir/test/partition_view_test.exs
index a255391..890cb88 100644
--- a/test/elixir/test/partition_view_test.exs
+++ b/test/elixir/test/partition_view_test.exs
@@ -286,6 +286,66 @@ defmodule ViewPartitionTest do
]
end
+ test "partition query can set query limits", context do
+ set_config({"query_server_config", "partition_query_limit", "2000"})
+
+ db_name = context[:db_name]
+ create_partition_docs(db_name)
+ create_partition_ddoc(db_name)
+
+ url = "/#{db_name}/_partition/foo/_design/mrtest/_view/some"
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 20
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 20
+
+ resp = Couch.get(url)
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 50
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2000
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 50
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2001
+ }
+ )
+
+ assert resp.status_code == 400
+ %{:body => %{"reason" => reason}} = resp
+ assert Regex.match?(~r/Limit is too large/, reason)
+
+ resp =
+ Couch.get(url,
+ query: %{
+ limit: 2000,
+ skip: 25
+ }
+ )
+
+ assert resp.status_code == 200
+ ids = get_ids(resp)
+ assert length(ids) == 25
+ end
+
test "include_design works correctly", context do
db_name = context[:db_name]