You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2025/10/31 17:51:52 UTC
(couchdb) 01/01: Log db+view for reduce overflow errors when enabled
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch enhance_reduce_limit-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 00c4f22b16da79bcb6cb77af59692688bab576ae
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Oct 30 10:53:18 2025 +0000
Log db+view for reduce overflow errors when enabled
---
share/server/views.js | 7 +------
src/couch/src/couch_proc_manager.erl | 3 ++-
src/couch/src/couch_query_servers.erl | 11 +++--------
src/fabric/include/fabric.hrl | 2 ++
src/fabric/src/fabric_view_reduce.erl | 30 ++++++++++++++++++++++++++++--
src/fabric/src/fabric_view_row.erl | 2 ++
6 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/share/server/views.js b/share/server/views.js
index b80c997d9..4b88b8946 100644
--- a/share/server/views.js
+++ b/share/server/views.js
@@ -44,12 +44,7 @@ var Views = (function() {
"input size:", input_length,
"output size:", reduce_length
].join(" ");
- if (State.query_config.reduce_limit === "log") {
- log("reduce_overflow_error: " + log_message);
- print("[true," + reduce_line + "]");
- } else {
- throw(["error", "reduce_overflow_error", log_message]);
- };
+ throw(["error", "reduce_overflow_error", log_message]);
} else {
print("[true," + reduce_line + "]");
}
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index aa538e23e..106aa2b04 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -26,7 +26,8 @@
reload/0,
terminate_stale_procs/0,
get_servers_from_env/1,
- native_query_server_enabled/0
+ native_query_server_enabled/0,
+ get_reduce_limit/0
]).
-export([
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
index ef9e28f9b..dae7ac60b 100644
--- a/src/couch/src/couch_query_servers.erl
+++ b/src/couch/src/couch_query_servers.erl
@@ -282,26 +282,21 @@ check_sum_overflow(InSize, OutSize, Sum) ->
Overflowed = OutSize > reduce_limit_threshold() andalso OutSize * reduce_limit_ratio() > InSize,
case config:get("query_server_config", "reduce_limit", "true") of
"true" when Overflowed ->
- Msg = log_sum_overflow(InSize, OutSize),
+ Msg = sum_overflow_msg(InSize, OutSize),
{[
{<<"error">>, <<"builtin_reduce_error">>},
{<<"reason">>, Msg}
]};
- "log" when Overflowed ->
- log_sum_overflow(InSize, OutSize),
- Sum;
_ ->
Sum
end.
-log_sum_overflow(InSize, OutSize) ->
+sum_overflow_msg(InSize, OutSize) ->
Fmt =
"Reduce output must shrink more rapidly: "
"input size: ~b "
"output size: ~b",
- Msg = iolist_to_binary(io_lib:format(Fmt, [InSize, OutSize])),
- couch_log:error(Msg, []),
- Msg.
+ iolist_to_binary(io_lib:format(Fmt, [InSize, OutSize])).
reduce_limit_threshold() ->
config:get_integer("query_server_config", "reduce_limit_threshold", 4906).
diff --git a/src/fabric/include/fabric.hrl b/src/fabric/include/fabric.hrl
index 6312741c2..4c0325d81 100644
--- a/src/fabric/include/fabric.hrl
+++ b/src/fabric/include/fabric.hrl
@@ -12,6 +12,8 @@
-record(collector, {
db_name=nil,
+ ddoc_name=nil,
+ view_name=nil,
query_args,
callback,
counters,
diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl
index 3206d01a4..a660abd6d 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -17,6 +17,7 @@
-include_lib("fabric/include/fabric.hrl").
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("couch/include/couch_db.hrl").
go(DbName, GroupId, View, Args, Callback, Acc0, VInfo) when is_binary(GroupId) ->
{ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []),
@@ -50,7 +51,7 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
Callback({error, insufficient_storage}, Acc);
{ok, Workers} ->
try
- go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
+ go2(DbName, Workers, DDoc, VName, VInfo, CoordArgs, Callback, Acc)
after
fabric_streams:cleanup(Workers)
end;
@@ -64,7 +65,7 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
rexi_monitor:stop(RexiMon)
end.
-go2(DbName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
+go2(DbName, Workers, DDoc, VName, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
#mrargs{limit = Limit, skip = Skip, keys = Keys, update_seq = UpdateSeq} = Args,
RedSrc = couch_mrview_util:extract_view_reduce(VInfo),
OsProc =
@@ -74,6 +75,8 @@ go2(DbName, Workers, {red, {_, Lang, View}, _} = VInfo, Args, Callback, Acc0) ->
end,
State = #collector{
db_name = DbName,
+ ddoc_name = DDoc#doc.id,
+ view_name = VName,
query_args = Args,
callback = Callback,
counters = fabric_dict:init(Workers, 0),
@@ -169,6 +172,19 @@ handle_message({meta, Meta0}, {Worker, From}, State) ->
end;
handle_message(#view_row{} = Row, {_, _} = Source, State) ->
handle_row(Row, Source, State);
+handle_message(
+ {view_row, #{value := {[{reduce_overflow_error, Msg}]}}} = Row, {_, _} = Source, State
+) ->
+ maybe_log_reduce_overflow(State, Msg),
+ handle_row(Row, Source, State);
+handle_message(
+ {view_row, #{value := {[{<<"error">>, <<"builtin_reduce_error">>}, {<<"reason">>, Msg}]}}} =
+ Row,
+ {_, _} = Source,
+ State
+) ->
+ maybe_log_reduce_overflow(State, Msg),
+ handle_row(Row, Source, State);
handle_message({view_row, #{}} = Row, {_, _} = Source, State) ->
handle_row(Row, Source, State);
handle_message(complete, Worker, #collector{counters = Counters0} = State) ->
@@ -180,6 +196,16 @@ handle_message(ddoc_updated, _Worker, State) ->
handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
+maybe_log_reduce_overflow(State, Msg) ->
+ case couch_proc_manager:get_reduce_limit() of
+ log ->
+ couch_log:warning("reduce_overflow from ~s/~s/_view/~s: ~s", [
+ State#collector.db_name, State#collector.ddoc_name, State#collector.view_name, Msg
+ ]);
+ _Else ->
+ ok
+ end.
+
os_proc_needed(<<"_", _/binary>>) -> false;
os_proc_needed(_) -> true.
diff --git a/src/fabric/src/fabric_view_row.erl b/src/fabric/src/fabric_view_row.erl
index 09dc3cff0..f7eafa18c 100644
--- a/src/fabric/src/fabric_view_row.erl
+++ b/src/fabric/src/fabric_view_row.erl
@@ -103,6 +103,8 @@ set_stats({view_row, #{} = Row}, Stats) ->
transform(#view_row{value = {[{reduce_overflow_error, Msg}]}}) ->
{row, [{key, null}, {id, error}, {value, reduce_overflow_error}, {reason, Msg}]};
+transform(#view_row{value = {[{<<"error">>,<<"builtin_reduce_error">>},{<<"reason">>, Msg}]}}) ->
+ {row, [{key, null}, {id, error}, {value, builtin_reduce_error}, {reason, Msg}]};
transform(#view_row{key = Key, id = reduced, value = Value}) ->
{row, [{key, Key}, {value, Value}]};
transform(#view_row{key = Key, id = undefined}) ->