You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2025/04/30 04:50:12 UTC
(couchdb) branch fix-query-limit-for-all-docs updated (a37253ce7 -> 42efaf88d)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch fix-query-limit-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git
omit a37253ce7 Fix query limit
add e8d1749c5 Update erlang version to their latest patch versions
add 1051692fb always remove holder when unloading
add d0fd17682 reopen an index if Lucene closed it for any reason
add 9ed0c4acc Merge pull request #5522 from apache/5521-handle-closed-index
new 42efaf88d Fix query limit
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a37253ce7)
\
N -- N -- N refs/heads/fix-query-limit-for-all-docs (42efaf88d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build-aux/Jenkinsfile.full | 4 +-
build-aux/Jenkinsfile.pr | 4 +-
.../org/apache/couchdb/nouveau/core/Index.java | 6 +
.../apache/couchdb/nouveau/core/IndexManager.java | 20 ++-
.../couchdb/nouveau/lucene9/Lucene9Index.java | 5 +
.../couchdb/nouveau/core/IndexManagerTest.java | 85 +++++++++
src/couch_mrview/src/couch_mrview_test_util.erl | 18 ++
.../test/eunit/couch_mrview_all_docs_tests.erl | 190 ++++++++++++++++++---
.../test/eunit/couch_mrview_show_list_tests.erl | 96 +++++++++++
src/fabric/test/eunit/fabric_tests.erl | 18 +-
10 files changed, 409 insertions(+), 37 deletions(-)
create mode 100644 nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java
create mode 100644 src/couch_mrview/test/eunit/couch_mrview_show_list_tests.erl
(couchdb) 01/01: Fix query limit
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch fix-query-limit-for-all-docs
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 42efaf88d9e65a08a18927021b435d0928e16c4a
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Apr 25 18:59:13 2025 -0400
Fix query limit
Previously, we set a default limit that was an effective infinity (2^28). It
seems back in the 32 bit days that was the Erlang's largest small integer [1].
However, that turned out to be too low and it surprised a user when it
truncated their all_docs output skipping some of the data. Fix that by
increasing the limit to a larger "infinity" (highest 64 bit Erlang small
integer [1]).
We did have a "query_limit" config parameter to customize the limit, however
that turned out to be broken and did not take effect when the user tried it for
all_docs, so fix that as well. Fix that and use a test to ensure the limit gets
reduced appropriately. To make the setting more user friendly, allow `infinity`
as the value.
Also, in the case of all_docs, we validated args and applied the limit check
twice: once in the coordinator and another time on each worker, which wasted
CPU resources and made things a bit confusing. To fix that, remove the
validation from the common worker code in couch_mrview and validate once:
either on the coordinator side, or local (port 5986) callback, right in the
http callback.
[1] https://www.erlang.org/doc/system/memory.html
Fix #5176
---
rel/overlay/etc/default.ini | 4 +-
src/chttpd/src/chttpd_db.erl | 8 +-
src/chttpd/test/eunit/chttpd_view_test.erl | 72 +++++++-
src/couch_mrview/include/couch_mrview.hrl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 8 +-
src/couch_mrview/src/couch_mrview_http.erl | 3 +-
src/couch_mrview/src/couch_mrview_show.erl | 3 +-
src/couch_mrview/src/couch_mrview_test_util.erl | 18 ++
src/couch_mrview/src/couch_mrview_util.erl | 14 +-
.../test/eunit/couch_mrview_all_docs_tests.erl | 190 ++++++++++++++++++---
.../test/eunit/couch_mrview_show_list_tests.erl | 96 +++++++++++
src/fabric/src/fabric.erl | 3 +-
src/fabric/src/fabric_util.erl | 2 +
src/fabric/test/eunit/fabric_tests.erl | 18 +-
14 files changed, 389 insertions(+), 53 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index dbed963b1..4ececaa52 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -513,8 +513,8 @@ authentication_db = _users
; Timeout for how long a response from a busy view group server can take.
; "infinity" is also a valid configuration value.
;group_info_timeout = 5000
-;query_limit = 268435456
-;partition_query_limit = 268435456
+;query_limit = infinity
+;partition_query_limit = infinity
; Configure what to use as the db tag when selecting design doc couchjs
; processes. The choices are:
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index b4c141f8c..a43baeae4 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -948,14 +948,12 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
all_docs_view(Req, Db, Keys, OP) ->
Args0 = couch_mrview_http:parse_body_and_query(Req, Keys),
- Args1 = Args0#mrargs{view_type = map},
- Args2 = fabric_util:validate_all_docs_args(Db, Args1),
- Args3 = set_namespace(OP, Args2),
- Args4 = set_include_sysdocs(OP, Req, Args3),
+ Args1 = set_namespace(OP, Args0),
+ Args2 = set_include_sysdocs(OP, Req, Args1),
Options = [{user_ctx, Req#httpd.user_ctx}],
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db = Db, req = Req, threshold = Max},
- {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args4),
+ {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args2),
{ok, Resp#vacc.resp}.
view_cb({row, Row} = Msg, Acc) ->
diff --git a/src/chttpd/test/eunit/chttpd_view_test.erl b/src/chttpd/test/eunit/chttpd_view_test.erl
index e478e6640..beb71f823 100644
--- a/src/chttpd/test/eunit/chttpd_view_test.erl
+++ b/src/chttpd/test/eunit/chttpd_view_test.erl
@@ -48,7 +48,8 @@
setup() ->
Hashed = couch_passwords:hash_admin_password(?PASS),
- ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false),
+ ok = config:set("admins", ?USER, ?b2l(Hashed), false),
+ ok = config:set("query_server_config", "query_limit", "100", false),
Db = ?tempdb(),
ok = create_db(Db),
ok = create_docs(Db),
@@ -57,7 +58,8 @@ setup() ->
teardown(Db) ->
ok = fabric:delete_db(Db),
- ok = config:delete("admins", ?USER, _Persist = false).
+ ok = config:delete("admins", ?USER, false),
+ ok = config:delete("query_server_config", "query_limit", false).
view_test_() ->
{
@@ -111,7 +113,11 @@ all_docs_test_() ->
?TDEF_FE(t_all_docs_with_key_non_existent_docs),
?TDEF_FE(t_all_docs_with_keys_non_existent_docs),
?TDEF_FE(t_all_docs_with_key_deleted_docs),
- ?TDEF_FE(t_all_docs_with_keys_deleted_docs)
+ ?TDEF_FE(t_all_docs_with_keys_deleted_docs),
+ ?TDEF_FE(t_all_docs_with_limit),
+ ?TDEF_FE(t_all_docs_with_skip),
+ ?TDEF_FE(t_all_docs_with_limit_and_skip),
+ ?TDEF_FE(t_all_docs_with_configured_query_limit)
]
}
}
@@ -370,6 +376,66 @@ t_view_map_reduce_with_keys_and_group(Db) ->
?assertEqual(Res, Res1),
?assertEqual(Code, Code1).
+t_all_docs_with_limit(Db) ->
+ {Code, Res} = req(get, url(Db, "_all_docs", "limit=2")),
+ ?assertEqual(200, Code),
+ ?assertMatch(
+ #{
+ <<"offset">> := 0,
+ <<"total_rows">> := 4,
+ <<"rows">> := [
+ #{<<"id">> := <<"_design/ddoc">>},
+ #{<<"id">> := <<"a">>}
+ ]
+ },
+ Res
+ ).
+
+t_all_docs_with_skip(Db) ->
+ {Code, Res} = req(get, url(Db, "_all_docs", "skip=2")),
+ ?assertEqual(200, Code),
+ ?assertMatch(
+ #{
+ <<"offset">> := 2,
+ <<"total_rows">> := 4,
+ <<"rows">> := [
+ #{<<"id">> := <<"b">>},
+ #{<<"id">> := <<"c">>}
+ ]
+ },
+ Res
+ ).
+
+t_all_docs_with_limit_and_skip(Db) ->
+ {Code, Res} = req(get, url(Db, "_all_docs", "skip=2&limit=1")),
+ ?assertEqual(200, Code),
+ ?assertMatch(
+ #{
+ <<"offset">> := 2,
+ <<"total_rows">> := 4,
+ <<"rows">> := [
+ #{<<"id">> := <<"b">>}
+ ]
+ },
+ Res
+ ).
+
+t_all_docs_with_configured_query_limit(Db) ->
+ config:set("query_server_config", "query_limit", "2", false),
+ {Code, Res} = req(get, url(Db, "_all_docs")),
+ ?assertEqual(200, Code),
+ ?assertMatch(
+ #{
+ <<"offset">> := 0,
+ <<"total_rows">> := 4,
+ <<"rows">> := [
+ #{<<"id">> := <<"_design/ddoc">>},
+ #{<<"id">> := <<"a">>}
+ ]
+ },
+ Res
+ ).
+
%%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%%
url(Db) ->
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index b31463c53..6ba226aad 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -54,7 +54,8 @@
view_states=nil
}).
--define(MAX_VIEW_LIMIT, 16#10000000).
+% Largest 64 bit small int: 2^59-1
+-define(MAX_VIEW_LIMIT, 16#7FFFFFFFFFFFFFF).
-record(mrargs, {
view_type,
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index b31938f74..037c35cdd 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -279,20 +279,20 @@ query_all_docs(Db, Args) ->
query_all_docs(Db, Args, fun default_cb/2, []).
query_all_docs(Db, Args, Callback, Acc) when is_list(Args) ->
- query_all_docs(Db, to_mrargs(Args), Callback, Acc);
+ Args1 = couch_mrview_util:validate_all_docs_args(Db, to_mrargs(Args)),
+ query_all_docs(Db, Args1, Callback, Acc);
query_all_docs(Db, Args0, Callback, Acc) ->
Sig = couch_util:with_db(Db, fun(WDb) ->
{ok, Info} = couch_db:get_db_info(WDb),
couch_index_util:hexsig(couch_hash:md5_hash(?term_to_bin(Info)))
end),
Args1 = Args0#mrargs{view_type = map},
- Args2 = couch_mrview_util:validate_all_docs_args(Db, Args1),
{ok, Acc1} =
- case Args2#mrargs.preflight_fun of
+ case Args1#mrargs.preflight_fun of
PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc);
_ -> {ok, Acc}
end,
- all_docs_fold(Db, Args2, Callback, Acc1).
+ all_docs_fold(Db, Args1, Callback, Acc1).
query_view(Db, DDoc, VName) ->
Args = #mrargs{extra = [{view_row_map, true}]},
diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl
index 23d8c738e..9645a7899 100644
--- a/src/couch_mrview/src/couch_mrview_http.erl
+++ b/src/couch_mrview/src/couch_mrview_http.erl
@@ -241,7 +241,8 @@ do_all_docs_req(Req, Db, Keys, NS) ->
),
IsAdmin = is_admin(Db),
Callback = get_view_callback(DbName, UsersDbName, IsAdmin),
- couch_mrview:query_all_docs(Db, Args, Callback, VAcc0)
+ ValidatedArgs = couch_mrview_util:validate_all_docs_args(Db, Args),
+ couch_mrview:query_all_docs(Db, ValidatedArgs, Callback, VAcc0)
end),
case is_record(Resp, vacc) of
true -> {ok, Resp#vacc.resp};
diff --git a/src/couch_mrview/src/couch_mrview_show.erl b/src/couch_mrview/src/couch_mrview_show.erl
index 7fec0c5cd..8cf8a3817 100644
--- a/src/couch_mrview/src/couch_mrview_show.erl
+++ b/src/couch_mrview/src/couch_mrview_show.erl
@@ -224,7 +224,8 @@ handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
Acc = #lacc{db = Db, req = Req, qserver = QServer, lname = LName},
case VName of
<<"_all_docs">> ->
- couch_mrview:query_all_docs(Db, Args, fun list_cb/2, Acc);
+ ValidatedArgs = couch_mrview_util:validate_all_docs_args(Db, Args),
+ couch_mrview:query_all_docs(Db, ValidatedArgs, fun list_cb/2, Acc);
_ ->
couch_mrview:query_view(Db, VDDoc, VName, Args, fun list_cb/2, Acc)
end
diff --git a/src/couch_mrview/src/couch_mrview_test_util.erl b/src/couch_mrview/src/couch_mrview_test_util.erl
index 918988ea3..96eac5531 100644
--- a/src/couch_mrview/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview/src/couch_mrview_test_util.erl
@@ -111,6 +111,21 @@ ddoc(red) ->
]}}
]}
);
+ddoc(list) ->
+ couch_doc:from_json_obj(
+ map_to_proplist(#{
+ <<"_id">> => <<"_design/lists">>,
+ <<"lists">> => #{
+ <<"list">> =>
+ <<"function(head, req){\n"
+ " send(\"H\");\n"
+ " var row;\n"
+ " while(row = getRow()) {send(row.key);}\n"
+ " return \"T\";\n"
+ " }">>
+ }
+ })
+ );
ddoc(Id) ->
couch_doc:from_json_obj(
{[
@@ -134,3 +149,6 @@ local_doc(Id) ->
{<<"val">>, Id}
]}
).
+
+map_to_proplist(#{} = Map) ->
+ jiffy:decode(jiffy:encode(Map)).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index a478685da..a5c81a073 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -522,11 +522,11 @@ apply_limit(ViewPartitioned, Args) ->
{false, _} -> "query_limit"
end,
- MaxLimit = config:get_integer(
- "query_server_config",
- LimitType,
- ?MAX_VIEW_LIMIT
- ),
+ MaxLimit =
+ case config:get("query_server_config", LimitType, "infinity") of
+ "infinity" -> ?MAX_VIEW_LIMIT;
+ Val -> list_to_integer(Val)
+ end,
% Set the highest limit possible if a user has not
% specified a limit
@@ -545,7 +545,7 @@ apply_limit(ViewPartitioned, Args) ->
end.
validate_all_docs_args(Db, Args0) ->
- Args = validate_args(Args0),
+ Args = validate_args(Args0#mrargs{view_type = map}),
DbPartitioned = couch_db:is_partitioned(Db),
Partition = get_extra(Args, partition),
@@ -557,7 +557,7 @@ validate_all_docs_args(Db, Args0) ->
Args1 = apply_limit(true, Args),
apply_all_docs_partition(Args1, Partition);
_ ->
- Args
+ apply_limit(false, Args)
end.
validate_args(Args) ->
diff --git a/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl
index 4de050ef4..b2a01ec72 100644
--- a/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl
+++ b/src/couch_mrview/test/eunit/couch_mrview_all_docs_tests.erl
@@ -15,16 +15,15 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(TIMEOUT, 1000).
-
setup() ->
+ ok = config:set("query_server_config", "query_limit", "infinity", false),
{ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map),
Db.
teardown(Db) ->
couch_db:close(Db),
couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]),
- ok.
+ config:delete("query_server_config", "query_limit", false).
all_docs_test_() ->
{
@@ -38,23 +37,31 @@ all_docs_test_() ->
fun setup/0,
fun teardown/1,
[
- fun should_query/1,
- fun should_query_with_range/1,
- fun should_query_with_non_string_key/1,
- fun should_query_with_non_string_keys/1,
- fun should_query_with_range_and_same_keys/1,
- fun raise_error_query_with_range_and_different_keys/1,
- fun should_query_with_range_rev/1,
- fun should_query_with_limit_and_skip/1,
- fun should_query_with_include_docs/1,
- fun should_query_empty_views/1
+ ?TDEF_FE(should_query),
+ ?TDEF_FE(should_query_with_range),
+ ?TDEF_FE(should_query_with_non_string_key),
+ ?TDEF_FE(should_query_with_non_string_keys),
+ ?TDEF_FE(should_query_with_range_and_same_keys),
+ ?TDEF_FE(raise_error_query_with_range_and_different_keys),
+ ?TDEF_FE(should_query_with_range_rev),
+ ?TDEF_FE(should_query_with_limit_and_skip),
+ ?TDEF_FE(should_query_with_include_docs),
+ ?TDEF_FE(should_query_empty_views),
+ ?TDEF_FE(http_query),
+ ?TDEF_FE(http_query_with_range),
+ ?TDEF_FE(http_query_with_non_string_key),
+ ?TDEF_FE(http_query_with_range_rev),
+ ?TDEF_FE(http_query_with_limit_and_skip),
+ ?TDEF_FE(http_query_with_limit_and_skip_and_query_limit),
+ ?TDEF_FE(http_query_with_query_limit_and_over_limit),
+ ?TDEF_FE(http_query_with_include_docs)
]
}
}
}.
should_query(Db) ->
- Result = run_query(Db, []),
+ Result = run_query(Db, ""),
Expect =
{ok, [
{meta, [{total, 11}, {offset, 0}]},
@@ -70,7 +77,7 @@ should_query(Db) ->
mk_row(<<"9">>, <<"1-558c8487d9aee25399a91b5d31d90fe2">>),
mk_row(<<"_design/bar">>, <<"1-a44e1dd1994a7717bf89c894ebd1f081">>)
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
should_query_with_range(Db) ->
Result = run_query(Db, [{start_key, <<"3">>}, {end_key, <<"5">>}]),
@@ -81,18 +88,18 @@ should_query_with_range(Db) ->
mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>),
mk_row(<<"5">>, <<"1-aaac5d460fd40f9286e57b9bf12e23d2">>)
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
should_query_with_non_string_key(Db) ->
Expect = {ok, [{meta, [{total, 11}, {offset, 0}]}]},
[
- ?_assertEqual(Expect, run_query(Db, [{start_key, Key}, {end_key, Key}]))
+ ?assertEqual(Expect, run_query(Db, [{start_key, Key}, {end_key, Key}]))
|| Key <- [1, a, [1, "2"], #{id => 1}]
].
should_query_with_non_string_keys(Db) ->
[
- ?_assertEqual(
+ ?assertEqual(
{ok, [
{meta, [{total, 11}, {offset, 0}]},
{row, [{id, error}, {key, Key}, {value, not_found}]}
@@ -109,14 +116,12 @@ should_query_with_range_and_same_keys(Db) ->
{meta, [{total, 11}, {offset, 0}]},
mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>)
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
raise_error_query_with_range_and_different_keys(Db) ->
Error = {query_parse_error, <<"`keys` is incompatible with `key`, `start_key` and `end_key`">>},
- [
- ?_assertThrow(Error, run_query(Db, [{keys, [<<"1">>]}, {start_key, <<"5">>}])),
- ?_assertThrow(Error, run_query(Db, [{keys, [<<"5">>]}, {start_key, <<"5">>}]))
- ].
+ ?assertThrow(Error, run_query(Db, [{keys, [<<"1">>]}, {start_key, <<"5">>}])),
+ ?assertThrow(Error, run_query(Db, [{keys, [<<"5">>]}, {start_key, <<"5">>}])).
should_query_with_range_rev(Db) ->
Result = run_query(Db, [
@@ -132,7 +137,7 @@ should_query_with_range_rev(Db) ->
mk_row(<<"4">>, <<"1-fcaf5852c08ffb239ac8ce16c409f253">>),
mk_row(<<"3">>, <<"1-7fbf84d56f8017880974402d60f5acd6">>)
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
should_query_with_limit_and_skip(Db) ->
Result = run_query(Db, [
@@ -147,7 +152,7 @@ should_query_with_limit_and_skip(Db) ->
mk_row(<<"6">>, <<"1-aca21c2e7bc5f8951424fcfc5d1209d8">>),
mk_row(<<"7">>, <<"1-4374aeec17590d82f16e70f318116ad9">>)
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
should_query_with_include_docs(Db) ->
Result = run_query(Db, [
@@ -167,7 +172,7 @@ should_query_with_include_docs(Db) ->
{meta, [{total, 11}, {offset, 8}]},
{row, [{id, <<"8">>}, {key, <<"8">>}, {value, Val}, {doc, Doc}]}
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
should_query_empty_views(Db) ->
Result = couch_mrview:query_view(Db, <<"_design/bar">>, <<"bing">>),
@@ -175,10 +180,141 @@ should_query_empty_views(Db) ->
{ok, [
{meta, [{total, 0}, {offset, 0}]}
]},
- ?_assertEqual(Expect, Result).
+ ?assertEqual(Expect, Result).
mk_row(Id, Rev) ->
{row, [{id, Id}, {key, Id}, {value, {[{rev, Rev}]}}]}.
run_query(Db, Opts) ->
couch_mrview:query_all_docs(Db, Opts).
+
+http_query(Db) ->
+ ?assertMatch(
+ [
+ #{<<"id">> := <<"1">>, <<"key">> := <<"1">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"10">>, <<"key">> := <<"10">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"2">>, <<"key">> := <<"2">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"3">>, <<"key">> := <<"3">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"4">>, <<"key">> := <<"4">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"5">>, <<"key">> := <<"5">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"6">>, <<"key">> := <<"6">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"7">>, <<"key">> := <<"7">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"8">>, <<"key">> := <<"8">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"9">>, <<"key">> := <<"9">>, <<"value">> := #{<<"rev">> := _}},
+ #{
+ <<"id">> := <<"_design/bar">>,
+ <<"key">> := <<"_design/bar">>,
+ <<"value">> := #{<<"rev">> := _}
+ }
+ ],
+ http_query(Db, "")
+ ).
+
+http_query_with_range(Db) ->
+ ?assertMatch(
+ [
+ #{<<"id">> := <<"3">>, <<"key">> := <<"3">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"4">>, <<"key">> := <<"4">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"5">>, <<"key">> := <<"5">>, <<"value">> := #{<<"rev">> := _}}
+ ],
+ http_query(Db, "?start_key=\"3\"&end_key=\"5\"")
+ ).
+
+http_query_with_non_string_key(Db) ->
+ {Code1, Res1} = http_req(Db, "?start_key=1&end_key=1"),
+ ?assertEqual(200, Code1),
+ ?assertEqual(#{<<"offset">> => 0, <<"rows">> => [], <<"total_rows">> => 11}, Res1),
+
+ {Code2, Res2} = http_req(Db, "?start_key=a&end_key=a"),
+ ?assertEqual(400, Code2),
+ ?assertEqual(#{<<"error">> => <<"bad_request">>, <<"reason">> => <<"invalid_json">>}, Res2),
+
+ {Code3, Res3} = http_req(Db, "?start_key=\[1,\"2\"\]&end_key=\[1,\"2\"\]"),
+ ?assertEqual(200, Code3),
+ ?assertEqual(#{<<"offset">> => 0, <<"rows">> => [], <<"total_rows">> => 11}, Res3),
+
+ {Code4, Res4} = http_req(Db, "?start_key={\"id\":1}&end_key={\"id\":1}"),
+ ?assertEqual(200, Code4),
+ ?assertEqual(#{<<"offset">> => 0, <<"rows">> => [], <<"total_rows">> => 11}, Res4).
+
+http_query_with_range_rev(Db) ->
+ Params = "?descending=true&start_key=\"5\"&end_key=\"3\"&include_end=true",
+ ?assertMatch(
+ [
+ #{<<"id">> := <<"5">>, <<"key">> := <<"5">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"4">>, <<"key">> := <<"4">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"3">>, <<"key">> := <<"3">>, <<"value">> := #{<<"rev">> := _}}
+ ],
+ http_query(Db, Params)
+ ).
+
+http_query_with_limit_and_skip(Db) ->
+ Params = "?start_key=\"2\"&limit=3&skip=3",
+ ?assertMatch(
+ [
+ #{<<"id">> := <<"5">>, <<"key">> := <<"5">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"6">>, <<"key">> := <<"6">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"7">>, <<"key">> := <<"7">>, <<"value">> := #{<<"rev">> := _}}
+ ],
+ http_query(Db, Params)
+ ).
+
+http_query_with_limit_and_skip_and_query_limit(Db) ->
+ config:set("query_server_config", "query_limit", "2", false),
+ Params = "?start_key=\"3\"&skip=2",
+ ?assertMatch(
+ [
+ #{<<"id">> := <<"5">>, <<"key">> := <<"5">>, <<"value">> := #{<<"rev">> := _}},
+ #{<<"id">> := <<"6">>, <<"key">> := <<"6">>, <<"value">> := #{<<"rev">> := _}}
+ ],
+ http_query(Db, Params)
+ ).
+
+http_query_with_query_limit_and_over_limit(Db) ->
+ config:set("query_server_config", "query_limit", "2", false),
+ {Code, Res} = http_req(Db, "?limit=3"),
+ ?assertEqual(400, Code),
+ ?assertMatch(
+ #{
+ <<"error">> := <<"query_parse_error">>,
+ <<"reason">> := <<"Limit is too large", _/binary>>
+ },
+ Res
+ ).
+
+http_query_with_include_docs(Db) ->
+ Params = "?start_key=\"8\"&end_key=\"8\"&limit=8&include_docs=true",
+ ?assertMatch(
+ [
+ #{
+ <<"id">> := Id,
+ <<"key">> := Id,
+ <<"value">> := #{<<"rev">> := Rev},
+ <<"doc">> := #{
+ <<"_id">> := Id,
+ <<"_rev">> := Rev,
+ <<"val">> := 8
+ }
+ }
+ ],
+ http_query(Db, Params)
+ ).
+
+db_url(Db) ->
+ DbName = couch_db:name(Db),
+ Addr = config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ ?b2l(DbName).
+
+http_req(DbName, Params) ->
+ Url = db_url(DbName) ++ "/_all_docs",
+ {ok, Code, _Headers, Body} = test_request:get(Url ++ Params),
+ Res = #{} = jiffy:decode(Body, [return_maps]),
+ {Code, Res}.
+
+http_query(DbName, Params) ->
+ Url = db_url(DbName) ++ "/_all_docs",
+ {ok, Code, _Headers, Body} = test_request:get(Url ++ Params),
+ ?assertEqual(200, Code),
+ #{<<"rows">> := Rows} = jiffy:decode(Body, [return_maps]),
+ Rows.
diff --git a/src/couch_mrview/test/eunit/couch_mrview_show_list_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_show_list_tests.erl
new file mode 100644
index 000000000..30a3af18f
--- /dev/null
+++ b/src/couch_mrview/test/eunit/couch_mrview_show_list_tests.erl
@@ -0,0 +1,96 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_show_list_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+setup() ->
+ ok = config:set("query_server_config", "query_limit", "infinity", false),
+ {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), list, 5),
+ Db.
+
+teardown(Db) ->
+ couch_db:close(Db),
+ couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]),
+ config:delete("query_server_config", "query_limit", false).
+
+list_test_() ->
+ {
+ "_list tests",
+ {
+ setup,
+ fun test_util:start_couch/0,
+ fun test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(http_query),
+ ?TDEF_FE(http_query_with_range),
+ ?TDEF_FE(http_query_with_non_string_key),
+ ?TDEF_FE(http_query_with_range_rev),
+ ?TDEF_FE(http_query_with_limit_and_skip),
+ ?TDEF_FE(http_query_with_limit_and_skip_and_query_limit),
+ ?TDEF_FE(http_query_with_query_limit_and_over_limit)
+ ]
+ }
+ }
+ }.
+
+http_query(Db) ->
+ ?assertEqual(<<"H12345_design/listsT">>, http_query(Db, "")).
+
+http_query_with_range(Db) ->
+ ?assertEqual(<<"H345T">>, http_query(Db, "?start_key=\"3\"&end_key=\"5\"")).
+
+http_query_with_non_string_key(Db) ->
+ {Code1, Res1} = http_req(Db, "?start_key=1&end_key=1"),
+ ?assertEqual(200, Code1),
+ ?assertEqual(<<"HT">>, Res1).
+
+http_query_with_range_rev(Db) ->
+ Params = "?descending=true&start_key=\"5\"&end_key=\"3\"&include_end=true",
+ ?assertEqual(<<"H543T">>, http_query(Db, Params)).
+
+http_query_with_limit_and_skip(Db) ->
+ Params = "?start_key=\"2\"&limit=3&skip=3",
+ ?assertEqual(<<"H5_design/listsT">>, http_query(Db, Params)).
+
+http_query_with_limit_and_skip_and_query_limit(Db) ->
+ config:set("query_server_config", "query_limit", "2", false),
+ Params = "?start_key=\"1\"&skip=2",
+ ?assertEqual(<<"H34T">>, http_query(Db, Params)).
+
+http_query_with_query_limit_and_over_limit(Db) ->
+ config:set("query_server_config", "query_limit", "2", false),
+ {Code, _} = http_req(Db, "?limit=3"),
+ ?assertEqual(400, Code).
+
+db_url(Db) ->
+ DbName = couch_db:name(Db),
+ Addr = config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ ?b2l(DbName).
+
+http_req(DbName, Params) ->
+ Url = db_url(DbName) ++ "/_design/lists/_list/list/_all_docs",
+ {ok, Code, _Headers, Body} = test_request:get(Url ++ Params),
+ {Code, Body}.
+
+http_query(DbName, Params) ->
+ Url = db_url(DbName) ++ "/_design/lists/_list/list/_all_docs",
+ {ok, Code, _Headers, Body} = test_request:get(Url ++ Params),
+ ?assertEqual(200, Code),
+ Body.
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 7394a1978..d552a387d 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -411,9 +411,10 @@ all_docs(DbName, Callback, Acc, QueryArgs) ->
) ->
{ok, any()} | {error, Reason :: term()}.
-all_docs(DbName, Options, Callback, Acc0, #mrargs{} = QueryArgs) when
+all_docs(DbName, Options, Callback, Acc0, #mrargs{} = QueryArgs0) when
is_function(Callback, 2)
->
+ QueryArgs = fabric_util:validate_all_docs_args(DbName, QueryArgs0),
fabric_view_all_docs:go(dbname(DbName), opts(Options), QueryArgs, Callback, Acc0);
%% @doc convenience function that takes a keylist rather than a record
%% @equiv all_docs(DbName, Callback, Acc0, kl_to_query_args(QueryArgs))
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index 4acb65c73..d0961533f 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -326,6 +326,8 @@ is_partitioned(DbName0) when is_binary(DbName0) ->
is_partitioned(Db) ->
couch_db:is_partitioned(Db).
+validate_all_docs_args(DbName, Args) when is_list(DbName) ->
+ validate_all_docs_args(list_to_binary(DbName), Args);
validate_all_docs_args(DbName, Args) when is_binary(DbName) ->
Shards = mem3:shards(fabric:dbname(DbName)),
Db = open_cluster_db(hd(Shards)),
diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl
index b2c235fd3..1ba5d1bc6 100644
--- a/src/fabric/test/eunit/fabric_tests.erl
+++ b/src/fabric/test/eunit/fabric_tests.erl
@@ -285,7 +285,8 @@ query_view_test_() ->
fun setup/0,
fun teardown/1,
[
- ?TDEF_FE(t_query_view_configuration)
+ ?TDEF_FE(t_query_view_configuration),
+ ?TDEF_FE(t_query_all_docs)
]
}.
@@ -305,6 +306,21 @@ t_query_view_configuration({_Ctx, DbName}) ->
meck:expect(fabric_view_map, go, Parameters, meck:val(fabric_view_map_results)),
?assertEqual(fabric_view_map_results, fabric:query_view(DbName, DDocName, ViewName)).
+t_query_all_docs({_Ctx, DbName}) ->
+ Cbk = fun
+ ({meta, _}, Acc) -> {ok, Acc};
+ ({row, Row}, Acc) -> {ok, [Row | Acc]};
+ (complete, Acc) -> {ok, Acc}
+ end,
+ {ok, Rows} = fabric:all_docs(binary_to_list(DbName), Cbk, [], [{limit, 2}]),
+ ?assertMatch(
+ [
+ [{id, <<"_design/foo">>} | _],
+ [{id, <<"_design/boo">>} | _]
+ ],
+ Rows
+ ).
+
fabric_all_dbs_test_() ->
{
foreach,