[couchdb] branch 3.x updated: bypass partition query limit for mango (#3114)

[wo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wohali pushed a commit to branch 3.x
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/3.x by this push:
     new bb869c1  bypass partition query limit for mango (#3114)
bb869c1 is described below

commit bb869c1c27354550309f025f98a49b193a492a1f
Author: Tony Sun <to...@gmail.com>
AuthorDate: Fri Aug 28 12:56:00 2020 -0700

    bypass partition query limit for mango (#3114)
    
    When partition_query_limit is set for couch_mrview, it limits how many
    docs can be scanned when executing partitioned queries. But this limits
    mango's doc scans internally. This leads to documents not being scanned
    to fulfill a query. This fixes:
    https://github.com/apache/couchdb/issues/2795
    
    Co-authored-by: Joan Touzet <wo...@users.noreply.github.com>
---
 src/couch_mrview/src/couch_mrview_util.erl |  9 +++--
 src/mango/src/mango_cursor_view.erl        |  6 +++-
 test/elixir/test/partition_mango_test.exs  | 53 ++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index e971720..d318a3f 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -425,9 +425,12 @@ validate_args(#mrst{} = State, Args0) ->
 
 
 apply_limit(ViewPartitioned, Args) ->
-    LimitType = case ViewPartitioned of
-        true -> "partition_query_limit";
-        false -> "query_limit"
+    Options = Args#mrargs.extra,
+    IgnorePQLimit = lists:keyfind(ignore_partition_query_limit, 1, Options),
+    LimitType = case {ViewPartitioned, IgnorePQLimit} of
+        {true, false} -> "partition_query_limit";
+        {true, _} -> "query_limit";
+        {false, _} -> "query_limit"
     end,
 
     MaxLimit = config:get_integer("query_server_config",
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 240ef50..68d7c3b 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -116,7 +116,11 @@ base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
         start_key = StartKey,
         end_key = EndKey,
         include_docs = true,
-        extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
+        extra = [
+            {callback, {?MODULE, view_cb}},
+            {selector, Selector},
+            {ignore_partition_query_limit, true}
+        ]
     }.
 
 
diff --git a/test/elixir/test/partition_mango_test.exs b/test/elixir/test/partition_mango_test.exs
index 992999f..9e4f1e7 100644
--- a/test/elixir/test/partition_mango_test.exs
+++ b/test/elixir/test/partition_mango_test.exs
@@ -547,6 +547,59 @@ defmodule PartitionMangoTest do
   end
 
   @tag :with_partitioned_db
+  test "partitioned query with query server config set", context do
+    db_name = context[:db_name]
+    create_partition_docs(db_name)
+    create_index(db_name, ["value"])
+
+    # this is to test that we bypass partition_query_limit for mango
+    set_config({"query_server_config", "partition_query_limit", "1"})
+
+    url = "/#{db_name}/_partition/foo/_find"
+
+    resp =
+      Couch.post(
+        url,
+        body: %{
+          selector: %{
+            value: %{
+              "$gte": 6,
+              "$lt": 16
+            }
+          },
+          limit: 3
+        }
+      )
+
+    assert resp.status_code == 200
+    partitions = get_partitions(resp)
+    assert length(partitions) == 3
+    assert_correct_partition(partitions, "foo")
+
+    %{:body => %{"bookmark" => bookmark}} = resp
+
+    resp =
+      Couch.post(
+        url,
+        body: %{
+          selector: %{
+            value: %{
+              "$gte": 6,
+              "$lt": 16
+            }
+          },
+          limit: 3,
+          bookmark: bookmark
+        }
+      )
+
+    assert resp.status_code == 200
+    partitions = get_partitions(resp)
+    assert length(partitions) == 2
+    assert_correct_partition(partitions, "foo")
+  end
+
+  @tag :with_partitioned_db
   test "global query uses global index", context do
     db_name = context[:db_name]
     create_partition_docs(db_name)