You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2024/10/08 13:09:30 UTC
(couchdb) 01/01: nouveau: Add ?top_n=X query parameter for facets
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch top_n
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5778d81590dd3a825bbddcd7e3073383ece61251
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Oct 8 14:09:15 2024 +0100
nouveau: Add ?top_n=X query parameter for facets
---
.../couchdb/nouveau/api/SearchRequestTest.java | 1 +
.../src/test/resources/fixtures/SearchRequest.json | 3 +-
src/nouveau/src/nouveau_fabric_search.erl | 62 +++++++++++++++++++---
src/nouveau/src/nouveau_httpd.erl | 15 ++++++
4 files changed, 72 insertions(+), 9 deletions(-)
diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java
index 01489b021..9269f11e8 100644
--- a/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java
+++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java
@@ -51,6 +51,7 @@ public class SearchRequestTest {
result.setQuery("*:*");
result.setLimit(10);
result.setCounts(List.of("bar"));
+ result.setTopN(5);
result.setRanges(Map.of("foo", List.of(new DoubleRange("0 to 100 inc", 0.0, true, 100.0, true))));
return result;
}
diff --git a/nouveau/src/test/resources/fixtures/SearchRequest.json b/nouveau/src/test/resources/fixtures/SearchRequest.json
index c588cc16b..2583fb2ad 100644
--- a/nouveau/src/test/resources/fixtures/SearchRequest.json
+++ b/nouveau/src/test/resources/fixtures/SearchRequest.json
@@ -5,6 +5,7 @@
"counts": [
"bar"
],
+ "top_n": 5,
"ranges": {
"foo": [
{
@@ -14,4 +15,4 @@
}
]
}
-}
\ No newline at end of file
+}
diff --git a/src/nouveau/src/nouveau_fabric_search.erl b/src/nouveau/src/nouveau_fabric_search.erl
index 5008df9d6..d1cb7f87b 100644
--- a/src/nouveau/src/nouveau_fabric_search.erl
+++ b/src/nouveau/src/nouveau_fabric_search.erl
@@ -37,15 +37,24 @@ go(DbName, GroupId, IndexName, QueryArgs0) when is_binary(GroupId) ->
go(DbName, #doc{} = DDoc, IndexName, QueryArgs0) ->
{ok, Index} = nouveau_util:design_doc_to_index(DbName, DDoc, IndexName),
Shards = get_shards(DbName, QueryArgs0),
- {PackedBookmark, #{limit := Limit, sort := Sort} = QueryArgs1} =
+ {PackedBookmark, #{limit := Limit, top_n := GlobalTopN, sort := Sort} = QueryArgs1} =
maps:take(bookmark, QueryArgs0),
+ %% scale TopN by number of shards so we don't clip a global topN
+ %% label just because it wasn't in the topN locally.
+ LocalTopN =
+ case GlobalTopN of
+ undefined -> undefined;
+ Val -> Val * length(Shards)
+ end,
Bookmark = nouveau_bookmark:unpack(DbName, PackedBookmark),
Counters0 = lists:map(
fun(#shard{} = Shard) ->
After = maps:get(Shard#shard.range, Bookmark, null),
Ref = rexi:cast(
Shard#shard.node,
- {nouveau_rpc, search, [Shard#shard.name, Index, QueryArgs1#{'after' => After}]}
+ {nouveau_rpc, search, [
+ Shard#shard.name, Index, QueryArgs1#{'after' => After, top_n => LocalTopN}
+ ]}
),
Shard#shard{ref = Ref}
end,
@@ -72,7 +81,7 @@ go(DbName, #doc{} = DDoc, IndexName, QueryArgs0) ->
of
{ok, SearchResults} ->
NewBookmark = nouveau_bookmark:update(DbName, Bookmark, SearchResults),
- {ok, simplify_hits(SearchResults#{bookmark => NewBookmark})};
+ {ok, top_n_facets(simplify_hits(SearchResults#{bookmark => NewBookmark}), GlobalTopN)};
{error, Reason} ->
{error, Reason}
after
@@ -126,10 +135,10 @@ merge_search_results(A, B, #state{} = State) ->
State#state.limit
),
<<"counts">> => merge_facets(
- maps:get(<<"counts">>, A, null), maps:get(<<"counts">>, B, null), State#state.limit
+ maps:get(<<"counts">>, A, null), maps:get(<<"counts">>, B, null)
),
<<"ranges">> => merge_facets(
- maps:get(<<"ranges">>, A, null), maps:get(<<"ranges">>, B, null), State#state.limit
+ maps:get(<<"ranges">>, A, null), maps:get(<<"ranges">>, B, null)
)
}.
@@ -212,11 +221,48 @@ convert_item(Item) ->
maps:get(<<"value">>, Item)
end.
-merge_facets(FacetsA, null, _Limit) ->
+top_n_facets(SearchResults, TopN) ->
+ After = SearchResults#{
+ <<"counts">> => top_n_facets(<<"counts">>, SearchResults, TopN),
+ <<"ranges">> => top_n_facets(<<"ranges">>, SearchResults, TopN)
+ },
+ After.
+
+top_n_facets(Key, SearchResults, TopN) ->
+ Facets = maps:get(Key, SearchResults, null),
+ top_n_facets_int(Facets, TopN).
+
+top_n_facets_int(null, _TopN) ->
+ null;
+top_n_facets_int(Facets, TopN) when is_map(Facets), is_integer(TopN), TopN > 0 ->
+ MapFun = fun(_, Group) ->
+ couch_log:notice("group~n~p~nsorted~n~p~ntruncated~p~n~n", [
+ Group,
+ lists:sort(fun facet_sort/2, maps:to_list(Group)),
+ lists:sublist(lists:sort(fun facet_sort/2, maps:to_list(Group)), TopN)
+ ]),
+ maps:from_list(
+ lists:sublist(
+ lists:sort(fun facet_sort/2, maps:to_list(Group)),
+ TopN
+ )
+ )
+ end,
+ maps:map(MapFun, Facets).
+
+%% sort by value (high to low), tie-break on label (lowest wins)
+facet_sort({_K1, V1}, {_K2, V2}) when V1 > V2 ->
+ true;
+facet_sort({_K1, V1}, {_K2, V2}) when V1 < V2 ->
+ false;
+facet_sort({K1, V}, {K2, V}) ->
+ couch_ejson_compare:less_json(K1, K2).
+
+merge_facets(FacetsA, null) ->
FacetsA;
-merge_facets(null, FacetsB, _Limit) ->
+merge_facets(null, FacetsB) ->
FacetsB;
-merge_facets(FacetsA, FacetsB, _Limit) ->
+merge_facets(FacetsA, FacetsB) ->
Combiner = fun(_, V1, V2) -> maps:merge_with(fun(_, V3, V4) -> V3 + V4 end, V1, V2) end,
maps:merge_with(Combiner, FacetsA, FacetsB).
diff --git a/src/nouveau/src/nouveau_httpd.erl b/src/nouveau/src/nouveau_httpd.erl
index 10917b3d4..d17ec9e2a 100644
--- a/src/nouveau/src/nouveau_httpd.erl
+++ b/src/nouveau/src/nouveau_httpd.erl
@@ -68,6 +68,7 @@ handle_search_req_int(#httpd{method = 'GET', path_parts = [_, _, _, _, IndexName
locale => chttpd:qs_value(Req, "locale"),
partition => chttpd:qs_value(Req, "partition"),
limit => chttpd:qs_value(Req, "limit"),
+ top_n => chttpd:qs_value(Req, "top_n"),
sort => chttpd:qs_value(Req, "sort"),
ranges => chttpd:qs_value(Req, "ranges"),
counts => chttpd:qs_value(Req, "counts"),
@@ -87,6 +88,7 @@ handle_search_req_int(
locale => maps:get(<<"locale">>, ReqBody, undefined),
partition => chttpd:qs_value(Req, "partition"),
limit => maps:get(<<"limit">>, ReqBody, undefined),
+ top_n => maps:get(<<"top_n">>, ReqBody, undefined),
sort => json_or_undefined(<<"sort">>, ReqBody),
ranges => json_or_undefined(<<"ranges">>, ReqBody),
counts => json_or_undefined(<<"counts">>, ReqBody),
@@ -227,6 +229,19 @@ validate_query_arg(update, undefined) ->
true;
validate_query_arg(update, Bool) when is_boolean(Bool) ->
Bool;
+validate_query_arg(top_n, undefined) ->
+ 10;
+validate_query_arg(top_n, TopN) when is_integer(TopN), TopN > 0 ->
+ TopN;
+validate_query_arg(top_n, TopN) when is_integer(TopN) ->
+ throw({query_parse_error, <<"top_n parameter must be greater than zero">>});
+validate_query_arg(top_n, TopN) when is_list(TopN) ->
+ try
+ list_to_integer(TopN)
+ catch
+ error:badarg ->
+ throw({query_parse_error, <<"top_n parameter must be an integer">>})
+ end;
validate_query_arg(update, "false") ->
false;
validate_query_arg(update, "true") ->