You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2024/06/22 05:13:30 UTC
(couchdb) branch main updated (39721bd44 -> f1fc2f012)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
from 39721bd44 Remove couch_io_logger
new e0f16c56e Make QuickJS memory limits match Spidermonkey
new cc4a28930 Make QuickJS dispatch respond with an error instead of throwing a null
new f1fc2f012 Improve QuickJS scanner plugin
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
share/server/dispatch-quickjs.js | 5 +-
src/couch/test/eunit/couch_js_tests.erl | 32 ----
src/couch_quickjs/c_src/couchjs.c | 2 -
.../src/couch_quickjs_scanner_plugin.erl | 175 ++++++++++++++-------
.../test/couch_quickjs_scanner_plugin_tests.erl | 47 +++++-
src/couch_scanner/include/couch_scanner_plugin.hrl | 2 +
6 files changed, 167 insertions(+), 96 deletions(-)
(couchdb) 03/03: Improve QuickJS scanner plugin
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f1fc2f0121f46f9d429297af5dabb93b31a65d6c
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Jun 21 20:45:20 2024 -0400
Improve QuickJS scanner plugin
Some improvements I noticed we could make after scanning a few clusters.
* Be more resilient to errors, so we log the errors but do not get do not get
stuck forever retrying to rescan the same db/ddoc/doc.
* Reduce the memory and doc batch size limits. Previous size and memory limit
were taken from the mrview index settings, but we're also dealing with VDUs,
filters, and other things, so to keep the engines from crashing or consuming
too many resources opt to reduce the limits a bit at the expense of possibly
running longer.
* Previously, if a view check crashed the JS process, and it died, then any
filter or VDU checks for the same DB will fail because the new JS processes
won't have the "taught" design documents. To fix this, if we notice the SM or
QJS process is dead, we respawn it, and then also load all the design docs.
* Since we're not building an actual view btree, don't set a reduce limit. We
send potentially a lot more data to the reduce functions, and we'd have a
higher chance of triggering the reduce limit exception. So then we just end up
showing a discrepancy in stack traces and reduce limit error format, instead
of validationg the actual output of the reduce functions.
* Since QuickJS is the new engine, and is mostly likely to show issues or
crashes, always try to use the Spidermonkey process first instead of the
QuickJS one. Otherwise if SM crashes we may not get a good signal if there is
some discrepancy between engines, or just a bad JS code in general that throws
errors anyway.
* Add a few more tests for some large reduce values and some odd formatting
around functions.
---
.../src/couch_quickjs_scanner_plugin.erl | 175 ++++++++++++++-------
.../test/couch_quickjs_scanner_plugin_tests.erl | 47 +++++-
src/couch_scanner/include/couch_scanner_plugin.hrl | 2 +
3 files changed, 164 insertions(+), 60 deletions(-)
diff --git a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
index c68313d43..82f18ef36 100644
--- a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
+++ b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl
@@ -43,9 +43,8 @@
max_shards = 4,
max_docs = 1000,
max_step = 1000,
- % These match defaults in couch_mrview_updater
- max_batch_items = 100,
- max_batch_size = 16777216
+ max_batch_items = 50,
+ max_batch_size = 8388608
}).
% DDoc fields
@@ -131,37 +130,17 @@ doc_id(#st{doc_cnt = C} = St, _DocId, _Db) ->
{ok, St#st{doc_cnt = C + 1}}.
doc(#st{} = St, Db, #doc{id = DocId} = Doc) ->
- #st{
- sid = SId,
- ddocs = DDocs,
- docs = Docs,
- docs_size = DocsSize,
- max_batch_items = MaxItems,
- max_batch_size = MaxSize
- } = St,
+ #st{sid = SId} = St,
JsonDoc = couch_query_servers:json_doc(Doc),
- DDocFun = fun(DDocId, #{} = DDoc) ->
- try
- Filters = maps:get(?FILTERS, DDoc, undefined),
- filter_doc_validate(St, DDocId, Filters, JsonDoc),
- VDU = maps:get(?VDU, DDoc, undefined),
- vdu_doc_validate(St, DDocId, VDU, JsonDoc)
- catch
- throw:{validate, Error} ->
- Meta = #{sid => SId, db => Db, ddoc => DDocId, doc => DocId},
- ?WARN("doc validation failed ~p", [Error], Meta)
- end
- end,
- maps:foreach(DDocFun, DDocs),
- DocsSize1 = DocsSize + ?term_size(JsonDoc),
- case DocsSize1 < MaxSize andalso length(Docs) < MaxItems of
- true ->
- St1 = St#st{docs = [JsonDoc | Docs], docs_size = DocsSize1},
- {ok, St1};
- false ->
- {_Db, St1} = maps:fold(fun views_validate/3, {Db, St}, DDocs),
- St2 = St1#st{docs = [], docs_size = 0},
- {ok, St2}
+ try
+ St1 = maybe_reset_and_teach_ddocs(St),
+ process_doc_filter_and_vdu(St1, Db, DocId, JsonDoc),
+ process_doc_views(St1, Db, JsonDoc)
+ catch
+ Tag:Err:Stack ->
+ Meta = #{sid => SId, db => Db, doc => DocId},
+ ?ERR("doc validation exception ~p:~p:~p", [Tag, Err, Stack], Meta),
+ {ok, St}
end.
db_closing(#st{docs = []} = St, _Db) ->
@@ -197,23 +176,67 @@ process_ddoc(#st{} = St, DbName, #doc{} = DDoc0) ->
views_load(St1, maps:get(?VIEWS, DDoc, undefined)),
filters_load(St1, maps:get(?FILTERS, DDoc, undefined)),
vdu_load(St1, maps:get(?VDU, DDoc, undefined)),
- reset_procs(St1),
- teach_ddoc_validate(St1, DDocId, DDoc),
- St1#st{ddocs = DDocs#{DDocId => DDoc}}
+ St2 = start_or_reset_procs(St1),
+ teach_ddoc_validate(St2, DDocId, DDoc),
+ St2#st{ddocs = DDocs#{DDocId => DDoc}}
catch
throw:{validate, Error} ->
Meta = #{sid => SId, db => DbName, ddoc => DDocId},
?WARN("ddoc validation failed ~p", [Error], Meta),
- St1
+ St1;
+ Tag:Err:Stack ->
+ Meta = #{sid => SId, db => DbName, ddoc => DDocId},
+ ?ERR("ddoc validation exception ~p:~p:~p", [Tag, Err, Stack], Meta)
end;
false ->
St
end.
+process_doc_filter_and_vdu(#st{} = St, Db, DocId, JsonDoc) ->
+ #st{sid = SId, ddocs = DDocs} = St,
+ DDocFun = fun(DDocId, #{} = DDoc) ->
+ try
+ Filters = maps:get(?FILTERS, DDoc, undefined),
+ filter_doc_validate(St, DDocId, Filters, JsonDoc),
+ VDU = maps:get(?VDU, DDoc, undefined),
+ vdu_doc_validate(St, DDocId, VDU, JsonDoc)
+ catch
+ throw:{validate, Error} ->
+ Meta = #{sid => SId, db => Db, ddoc => DDocId, doc => DocId},
+ ?WARN("doc validation failed ~p", [Error], Meta)
+ end
+ end,
+ try
+ maps:foreach(DDocFun, DDocs)
+ catch
+ Tag:Err:Stack ->
+ Meta = #{sid => SId, db => Db, doc => DocId},
+ ?ERR("Exception when validating doc ~p:~p:~p", [Tag, Err, Stack], Meta)
+ end.
+
+process_doc_views(#st{} = St, Db, JsonDoc) ->
+ #st{
+ ddocs = DDocs,
+ docs = Docs,
+ docs_size = DocsSize,
+ max_batch_items = MaxItems,
+ max_batch_size = MaxSize
+ } = St,
+ DocsSize1 = DocsSize + ?term_size(JsonDoc),
+ case DocsSize1 < MaxSize andalso length(Docs) < MaxItems of
+ true ->
+ St1 = St#st{docs = [JsonDoc | Docs], docs_size = DocsSize1},
+ {ok, St1};
+ false ->
+ {_Db, St1} = maps:fold(fun views_validate/3, {Db, St}, DDocs),
+ St2 = St1#st{docs = [], docs_size = 0},
+ {ok, St2}
+ end.
+
views_validate(DDocId, #{?VIEWS := Views} = DDoc, {Db, #st{} = St0}) when
map_size(Views) > 0
->
- St = reset_procs(St0),
+ St = start_or_reset_procs(St0),
#st{sid = SId, docs = Docs} = St,
try
lib_load(St, maps:get(?LIB, DDoc, undefined)),
@@ -226,6 +249,10 @@ views_validate(DDocId, #{?VIEWS := Views} = DDoc, {Db, #st{} = St0}) when
throw:{validate, Error} ->
Meta = #{sid => SId, db => Db, ddoc => DDocId},
?WARN("view validation failed ~p", [Error], Meta),
+ {Db, St};
+ Tag:Err:Stack ->
+ Meta = #{sid => SId, db => Db, ddoc => DDocId},
+ ?ERR("view validation exception ~p:~p:~p", [Tag, Err, Stack], Meta),
{Db, St}
end;
views_validate(_DDocId, #{} = _DDoc, {Db, #st{} = St}) ->
@@ -235,8 +262,8 @@ views_validate(_DDocId, #{} = _DDoc, {Db, #st{} = St}) ->
mapred_fold({Props = [_ | _]} = Doc, {ViewList = [_ | _], #st{} = St}) ->
#st{qjs_proc = Qjs, sm_proc = Sm} = St,
DocId = couch_util:get_value(<<"_id">>, Props),
- QjsMapRes = map_doc(Qjs, Doc),
SmMapRes = map_doc(Sm, Doc),
+ QjsMapRes = map_doc(Qjs, Doc),
case QjsMapRes == SmMapRes of
true -> ok;
false -> throw({validate, {map_doc, DocId, QjsMapRes, SmMapRes}})
@@ -252,8 +279,8 @@ mapred_fold({Props = [_ | _]} = Doc, {ViewList = [_ | _], #st{} = St}) ->
{ViewList, St}.
reset_per_db_state(#st{qjs_proc = QjsProc, sm_proc = SmProc} = St) ->
- proc_stop(QjsProc),
proc_stop(SmProc),
+ proc_stop(QjsProc),
St#st{
ddocs = #{},
docs = [],
@@ -263,8 +290,8 @@ reset_per_db_state(#st{qjs_proc = QjsProc, sm_proc = SmProc} = St) ->
}.
start_or_reset_procs(#st{} = St) ->
- St1 = start_or_reset_qjs_proc(St),
- start_or_reset_sm_proc(St1).
+ St1 = start_or_reset_sm_proc(St),
+ start_or_reset_qjs_proc(St1).
start_or_reset_qjs_proc(#st{qjs_proc = undefined} = St) ->
Cmd = couch_quickjs:mainjs_cmd(),
@@ -277,6 +304,9 @@ start_or_reset_qjs_proc(#st{qjs_proc = #proc{} = Proc} = St) ->
true = proc_reset(Proc),
St
catch
+ exit:{noproc, _} ->
+ proc_stop(Proc),
+ start_or_reset_qjs_proc(St#st{qjs_proc = undefined});
Tag:Err ->
?WARN("failed to reset QuickJS proc ~p:~p", [Tag, Err]),
proc_stop(Proc),
@@ -294,6 +324,9 @@ start_or_reset_sm_proc(#st{sm_proc = #proc{} = Proc} = St) ->
true = proc_reset(Proc),
St
catch
+ exit:{noproc, _} ->
+ proc_stop(Proc),
+ start_or_reset_sm_proc(St#st{sm_proc = undefined});
Tag:Err ->
?WARN("failed to reset Spidermonkey proc ~p:~p", [Tag, Err]),
proc_stop(Proc),
@@ -303,8 +336,8 @@ start_or_reset_sm_proc(#st{sm_proc = #proc{} = Proc} = St) ->
lib_load(#st{}, undefined) ->
ok;
lib_load(#st{qjs_proc = Qjs, sm_proc = Sm}, #{} = Lib) ->
- QjsRes = add_lib(Qjs, Lib),
SmRes = add_lib(Sm, Lib),
+ QjsRes = add_lib(Qjs, Lib),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {add_lib, QjsRes, SmRes}})
@@ -325,8 +358,8 @@ view_load(#st{} = St, Name, View) ->
add_fun_load(#st{}, _, undefined) ->
ok;
add_fun_load(#st{qjs_proc = Qjs, sm_proc = Sm}, Name, Src) ->
- QjsRes = add_fun(Qjs, Src),
SmRes = add_fun(Sm, Src),
+ QjsRes = add_fun(Qjs, Src),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {add_fun, Name, QjsRes, SmRes}})
@@ -343,16 +376,16 @@ reduce_filter_map({{_Name, #{}}, _KVs}) ->
view_reduce_validate(#st{} = St, ReduceKVs) ->
#st{qjs_proc = Qjs, sm_proc = Sm} = St,
RedFun = fun({Name, RedSrc, KVs}) ->
- QjsRedRes = reduce(Qjs, RedSrc, KVs),
SmRedRes = reduce(Sm, RedSrc, KVs),
+ QjsRedRes = reduce(Qjs, RedSrc, KVs),
case QjsRedRes == SmRedRes of
true -> ok;
false -> throw({validate, {reduce, Name, QjsRedRes, SmRedRes}})
end,
case QjsRedRes of
[true, [_ | _] = RedVals] ->
- QjsRRedRes = rereduce(Qjs, RedSrc, RedVals),
SmRRedRes = rereduce(Sm, RedSrc, RedVals),
+ QjsRRedRes = rereduce(Qjs, RedSrc, RedVals),
case QjsRRedRes == SmRRedRes of
true -> ok;
false -> throw({validate, {rereduce, Name, QjsRRedRes, SmRRedRes}})
@@ -370,8 +403,8 @@ filters_load(#st{} = St, #{} = Filters) ->
maps:foreach(Fun, Filters).
filter_load(#st{qjs_proc = Qjs, sm_proc = Sm}, Name, Filter) ->
- QjsRes = add_fun(Qjs, Filter),
SmRes = add_fun(Sm, Filter),
+ QjsRes = add_fun(Qjs, Filter),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {filter, Name, QjsRes, SmRes}})
@@ -382,8 +415,8 @@ filter_doc_validate(#st{}, _, undefined, _) ->
filter_doc_validate(#st{} = St, DDocId, #{} = Filters, Doc) ->
#st{qjs_proc = Qjs, sm_proc = Sm} = St,
Fun = fun(FName, _) ->
- QjsRes = filter_doc(Qjs, DDocId, FName, Doc),
SmRes = filter_doc(Sm, DDocId, FName, Doc),
+ QjsRes = filter_doc(Qjs, DDocId, FName, Doc),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {filter_doc, FName, QjsRes, SmRes}})
@@ -394,8 +427,8 @@ filter_doc_validate(#st{} = St, DDocId, #{} = Filters, Doc) ->
vdu_load(#st{}, undefined) ->
ok;
vdu_load(#st{qjs_proc = Qjs, sm_proc = Sm}, VDU) ->
- QjsRes = add_fun(Qjs, VDU),
SmRes = add_fun(Sm, VDU),
+ QjsRes = add_fun(Qjs, VDU),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {vdu, QjsRes, SmRes}})
@@ -406,25 +439,46 @@ vdu_doc_validate(#st{}, _DDocId, undefined, _Doc) ->
ok;
vdu_doc_validate(#st{} = St, DDocId, _VDU, Doc) ->
#st{qjs_proc = Qjs, sm_proc = Sm} = St,
- QjsRes = vdu_doc(Qjs, DDocId, Doc),
SmRes = vdu_doc(Sm, DDocId, Doc),
+ QjsRes = vdu_doc(Qjs, DDocId, Doc),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {vdu_doc, QjsRes, SmRes}})
end.
teach_ddoc_validate(#st{qjs_proc = Qjs, sm_proc = Sm}, DDocId, DDoc) ->
- QjsRes = teach_ddoc(Qjs, DDocId, DDoc),
SmRes = teach_ddoc(Sm, DDocId, DDoc),
+ QjsRes = teach_ddoc(Qjs, DDocId, DDoc),
case QjsRes == SmRes of
true -> ok;
false -> throw({validate, {teach_ddoc, DDocId, QjsRes, SmRes}})
end.
-reset_procs(#st{qjs_proc = #proc{} = QjsProc, sm_proc = #proc{} = SmProc} = St) ->
- true = proc_reset(QjsProc),
- true = proc_reset(SmProc),
- St.
+% During view validation if a JS process crashes and exists the next filter and
+% vdu validation for the same db will fail as the "taught" design documents
+% will not be in loaded in the JS process ddoc cache. To make sure we continue
+% validating, try to restart the process and reload all the ddocs.
+%
+maybe_reset_and_teach_ddocs(#st{ddocs = DDocs} = St) ->
+ St1 =
+ case is_proc_alive(St#st.sm_proc) of
+ true ->
+ St;
+ false ->
+ StSm = start_or_reset_sm_proc(St),
+ FunSm = fun(DDocId, DDoc) -> teach_ddoc(StSm#st.sm_proc, DDocId, DDoc) end,
+ maps:foreach(FunSm, DDocs),
+ StSm
+ end,
+ case is_proc_alive(St#st.qjs_proc) of
+ true ->
+ St1;
+ false ->
+ StQjs = start_or_reset_qjs_proc(St1),
+ FunQjs = fun(DDocId, DDoc) -> teach_ddoc(StQjs#st.qjs_proc, DDocId, DDoc) end,
+ maps:foreach(FunQjs, DDocs),
+ StQjs
+ end.
% Proc commands
@@ -449,17 +503,19 @@ add_fun(#proc{} = Proc, <<_/binary>> = FunSrc) ->
prompt(Proc, [<<"add_fun">>, FunSrc]).
filter_doc(#proc{} = Proc, DDocId, FName, {[_ | _]} = Doc) ->
- prompt(Proc, [<<"ddoc">>, DDocId, [<<"filters">>, FName], [[Doc], #{}]]).
+ % Add a mock request object so param access doesn't throw a TypeError
+ MockReq = #{<<"query">> => #{}},
+ prompt(Proc, [<<"ddoc">>, DDocId, [<<"filters">>, FName], [[Doc], MockReq]]).
vdu_doc(#proc{} = Proc, DDocId, {[_ | _]} = Doc) ->
prompt(Proc, [<<"ddoc">>, DDocId, [<<"validate_doc_update">>], [Doc, Doc]]).
teach_ddoc(#proc{} = Proc, DDocId, DDoc) ->
- true = prompt(Proc, [<<"ddoc">>, <<"new">>, DDocId, DDoc]).
+ prompt(Proc, [<<"ddoc">>, <<"new">>, DDocId, DDoc]).
proc_reset(#proc{} = Proc) ->
Timeout = config:get_integer("couchdb", "os_process_timeout", 5000),
- Cfg = [{<<"reduce_limit">>, true}, {<<"timeout">>, Timeout}],
+ Cfg = [{<<"timeout">>, Timeout}],
Result = prompt(Proc, [<<"reset">>, {Cfg}]),
proc_set_timeout(Proc, Timeout),
Result.
@@ -468,6 +524,9 @@ proc_reset(#proc{} = Proc) ->
% Proc utils
+is_proc_alive(#proc{pid = Pid}) ->
+ is_process_alive(Pid).
+
proc(Pid) ->
#proc{
pid = Pid,
diff --git a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
index de6bfe806..9f3489180 100644
--- a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
+++ b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl
@@ -53,6 +53,7 @@ setup() ->
ok = add_doc(DbName, ?DOC4, #{a => w}),
ok = add_doc(DbName, ?DOC5, #{a => u}),
config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false),
+ reset_stats(),
{Ctx, DbName}.
teardown({Ctx, DbName}) ->
@@ -260,6 +261,22 @@ t_empty_ddoc({_, DbName}) ->
ok
end.
+reset_stats() ->
+ Counters = [
+ [couchdb, query_server, process_error_exits],
+ [couchdb, query_server, process_errors],
+ [couchdb, query_server, process_exits]
+ ],
+ [reset_counter(C) || C <- Counters].
+
+reset_counter(Counter) ->
+ case couch_stats:sample(Counter) of
+ 0 ->
+ ok;
+ N when is_integer(N), N > 0 ->
+ couch_stats:decrement_counter(Counter, N)
+ end.
+
config_delete_section(Section) ->
[config:delete(K, V, false) || {K, V} <- config:get(Section)].
@@ -306,6 +323,12 @@ ddoc_filter(Doc) ->
" return true;\n"
" }\n"
"}"
+ >>,
+ f_use_req => <<
+ "function (doc, req) {\n"
+ " if (doc.a == req.query.foo) {return true;} \n"
+ " return false; \n"
+ "}\n"
>>
}
}.
@@ -313,7 +336,7 @@ ddoc_filter(Doc) ->
ddoc_view(Doc) ->
Doc#{
views => #{
- v => #{
+ v1 => #{
map => <<
"function(doc) {\n"
" if(doc.a == 'x') {\n"
@@ -342,6 +365,26 @@ ddoc_view(Doc) ->
" }\n"
"}"
>>
- }
+ },
+ v_large_reduce => #{
+ map => <<
+ "function(doc) {\n"
+ " if(doc.a == 'x') {\n"
+ " r = doc.a.search(/(x+)/); emit(r, RegExp.$1)\n"
+ " } else {\n"
+ " for(i=0;i<10000;i++) {emit(doc.a, doc.a)};\n"
+ " }\n"
+ "}"
+ >>,
+ reduce => <<
+ "function(ks, vs, rereduce) {\n"
+ " return {'ks1': ks, 'ks2': k2, 'vs1': v2, 'v2':v2};\n"
+ "}"
+ >>
+ },
+ v_expr_fun1 => #{map => <<"(function(doc) {emit(1,2)})\n">>},
+ v_expr_fun2 => #{map => <<"(function(doc) {emit(3,4)});">>},
+ v_expr_fun3 => #{map => <<"y=9;\n(function(doc) {emit(5,y)})">>},
+ v_expr_fun4 => #{map => <<"x=10; function(doc) {emit(x,6)}\n">>}
}
}.
diff --git a/src/couch_scanner/include/couch_scanner_plugin.hrl b/src/couch_scanner/include/couch_scanner_plugin.hrl
index 76d7383d3..c2d0cc886 100644
--- a/src/couch_scanner/include/couch_scanner_plugin.hrl
+++ b/src/couch_scanner/include/couch_scanner_plugin.hrl
@@ -20,5 +20,7 @@
-define(INFO(), ?LOG(info, "", [], #{})).
-define(WARN(FMT, ARGS), ?LOG(warning, FMT, ARGS, #{})).
-define(WARN(FMT, ARGS, META), ?LOG(warning, FMT, ARGS, META)).
+-define(ERR(FMT, ARGS), ?LOG(error, FMT, ARGS, #{})).
+-define(ERR(FMT, ARGS, META), ?LOG(error, FMT, ARGS, META)).
-define(DEBUG(FMT, ARGS), ?LOG(debug, FMT, ARGS, #{fn => ?FUNCTION_NAME})).
-define(DEBUG(FMT, ARGS, META), ?LOG(debug, FMT, ARGS, maps:merge(META, #{fn => ?FUNCTION_NAME}))).
(couchdb) 01/03: Make QuickJS memory limits match Spidermonkey
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e0f16c56ea7d126d98b591a918ad07b599097eae
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Jun 21 20:40:37 2024 -0400
Make QuickJS memory limits match Spidermonkey
Spidermonkey only sets the stack limit size, so do the same for QuickJS.
This changes how the oom test behaves and since already have two oom tests
remove the flaky one.
---
src/couch/test/eunit/couch_js_tests.erl | 32 --------------------------------
src/couch_quickjs/c_src/couchjs.c | 2 --
2 files changed, 34 deletions(-)
diff --git a/src/couch/test/eunit/couch_js_tests.erl b/src/couch/test/eunit/couch_js_tests.erl
index c56ff69fc..764844e63 100644
--- a/src/couch/test/eunit/couch_js_tests.erl
+++ b/src/couch/test/eunit/couch_js_tests.erl
@@ -29,7 +29,6 @@ couch_js_test_() ->
?TDEF(should_replace_broken_utf16),
?TDEF(should_allow_js_string_mutations),
?TDEF(should_bump_timing_and_call_stats),
- ?TDEF(should_exit_on_oom, 60),
?TDEF(should_exit_on_internal_error, 60)
])
}
@@ -342,23 +341,6 @@ should_bump_timing_and_call_stats(_) ->
couch_query_servers:ret_os_process(Proc).
-%% erlfmt-ignore
-should_exit_on_oom(_) ->
- config:set("couchdb", "os_process_timeout", "15000", _Persist = false),
- Src = <<"
- var state = [];
- function(doc) {
- var val = \"0123456789ABCDEF\";
- for(var i = 0; i < 665535; i++) {
- state.push([val, val]);
- emit(null, null);
- }
- }
- ">>,
- Proc = couch_query_servers:get_os_process(<<"javascript">>),
- true = prompt(Proc, [<<"add_fun">>, Src]),
- trigger_oom(Proc).
-
%% erlfmt-ignore
should_exit_on_internal_error(_) ->
% A different way to trigger OOM which previously used to
@@ -400,20 +382,6 @@ should_exit_on_internal_error(_) ->
end,
?assert(couch_stats:sample([couchdb, query_server, process_errors]) > 0).
-trigger_oom(Proc) ->
- Status =
- try
- prompt(Proc, [<<"map_doc">>, <<"{}">>]),
- continue
- catch
- throw:{os_process_error, {exit_status, 1}} ->
- done
- end,
- case Status of
- continue -> trigger_oom(Proc);
- done -> ok
- end.
-
sample_time(Stat) ->
couch_stats:sample([couchdb, query_server, time, Stat]).
diff --git a/src/couch_quickjs/c_src/couchjs.c b/src/couch_quickjs/c_src/couchjs.c
index 434d264e7..7958acda5 100644
--- a/src/couch_quickjs/c_src/couchjs.c
+++ b/src/couch_quickjs/c_src/couchjs.c
@@ -373,7 +373,6 @@ static JSContext* new_cx(const couch_args* args) {
BAIL("Could not create JSRuntime");
}
- JS_SetMemoryLimit(rt, args->stack_size);
JS_SetMaxStackSize(rt, args->stack_size);
cx = JS_NewContextRaw(rt);
@@ -501,4 +500,3 @@ int main(int argc, const char* argv[])
return EXIT_SUCCESS;
}
-
(couchdb) 02/03: Make QuickJS dispatch respond with an error instead of throwing a null
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cc4a289309d1fd0519fa3f95392add8b1b3ad6d6
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Jun 21 20:42:01 2024 -0400
Make QuickJS dispatch respond with an error instead of throwing a null
For some internal errors, especially when bumping into memory limits, a null
exception is thrown, instead of rethrowing try to behave like the SM loop.js
and other error handling by returning an error line and then indicating we
shouldn't continue. This way process crashes will be reserved for segfaults and
such.
---
share/server/dispatch-quickjs.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/share/server/dispatch-quickjs.js b/share/server/dispatch-quickjs.js
index fc159f4c9..78ece8b59 100644
--- a/share/server/dispatch-quickjs.js
+++ b/share/server/dispatch-quickjs.js
@@ -121,8 +121,9 @@ var DDoc = (function() {
function handleError(e) {
if (e === null) {
// internal error, another possibility when out of memory
- // nothing to do except rethrow and let main.c catch it and exit(1)
- throw(null);
+ // we tell the client it was a fatal error by dying
+ respond(["error", "internal_error", null]);
+ return false;
}
const type = e[0];
if (type == "fatal") {