You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2025/09/26 22:11:40 UTC
(couchdb) branch resume-after-skip created (now d91b3cfc1)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch resume-after-skip
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at d91b3cfc1 Reschedule scanner plugins if they return skip on start or resume
This branch includes the following new commits:
new 711056939 Handle plugin stops without crashing
new d91b3cfc1 Reschedule scanner plugins if they return skip on start or resume
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb) 01/02: Handle plugin stops without crashing
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch resume-after-skip
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 71105693971d76c1da5036b93c3e6b87e5aaf6d1
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Sep 26 17:14:13 2025 -0400
Handle plugin stops without crashing
---
src/couch_scanner/src/couch_scanner_server.erl | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/couch_scanner/src/couch_scanner_server.erl b/src/couch_scanner/src/couch_scanner_server.erl
index 7176173d2..2ec1ac540 100644
--- a/src/couch_scanner/src/couch_scanner_server.erl
+++ b/src/couch_scanner/src/couch_scanner_server.erl
@@ -252,6 +252,8 @@ sched_exit_update(Id, #sched{} = Sched, {shutdown, reset}) ->
couch_log:warning("~p : resetting plugin ~s", [?MODULE, Id]),
couch_scanner_checkpoint:reset(Id),
Sched#sched{start_time = 0, error_count = 0, reschedule = tsec()};
+sched_exit_update(_Id, #sched{} = Sched, {shutdown, stop}) ->
+ Sched#sched{start_time = 0, error_count = 0};
sched_exit_update(Id, #sched{} = Sched, Norm) when
Norm == shutdown; Norm == normal
->
(couchdb) 02/02: Reschedule scanner plugins if they return skip on start or resume
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch resume-after-skip
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d91b3cfc188f4b0d65df72c134e6ca7b0d8405fa
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Sep 26 17:58:47 2025 -0400
Reschedule scanner plugins if they return skip on start or resume
Previously, if a plugin returned `skip` on `start/2` or `resume/2`, then the
plugin would stop and it wouldn't be scheduler to run again, even if it has a
"repeat" time interval set.
The new behavior on `skip` is to schedule the plugin to run again if plugin is
configured to repeat it's execution. This way, a plugin may react to a
temporary condition (feature disabled, some missing or crashed dependency) with
a `skip`, and then it will get a chance to retry later hoping the conditions
will change and it will continue executing.
---
src/couch_scanner/src/couch_scanner_plugin.erl | 29 ++++++++++++++++------
.../test/eunit/couch_scanner_test.erl | 20 +++++++++++++++
2 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/src/couch_scanner/src/couch_scanner_plugin.erl b/src/couch_scanner/src/couch_scanner_plugin.erl
index 1ea6d57e4..fe77e8e9d 100644
--- a/src/couch_scanner/src/couch_scanner_plugin.erl
+++ b/src/couch_scanner/src/couch_scanner_plugin.erl
@@ -246,7 +246,7 @@ init_from_checkpoint(#st{} = St) ->
<<"start_sec">> := StartSec
} ->
Now = tsec(),
- PSt = resume_callback(Cbks, SId, EJsonPSt),
+ PSt = resume_callback(Mod, Cbks, SId, EJsonPSt),
St#st{
pst = PSt,
cursor = Cur,
@@ -502,20 +502,33 @@ start_callback(Mod, Cbks, Now, ScanId, LastStartSec, #{} = EJson) when
TSec when is_integer(TSec), TSec =< Now ->
#{start := StartCbk} = Cbks,
case StartCbk(ScanId, EJson) of
- {ok, PSt} -> PSt;
- skip -> exit_resched(infinity);
- reset -> exit_resched(reset)
+ {ok, PSt} ->
+ PSt;
+ skip ->
+ % If plugin skipped start, count this as an attempt and
+ % and reschedule for the future.
+ SkipReschedTSec = schedule_time(Mod, Now, Now),
+ exit_resched(SkipReschedTSec);
+ reset ->
+ exit_resched(reset)
end;
TSec when is_integer(TSec), TSec > Now ->
exit_resched(TSec)
end.
-resume_callback(#{} = Cbks, SId, #{} = EJsonPSt) when is_binary(SId) ->
+resume_callback(Mod, #{} = Cbks, SId, #{} = EJsonPSt) when is_binary(SId) ->
#{resume := ResumeCbk} = Cbks,
case ResumeCbk(SId, EJsonPSt) of
- {ok, PSt} -> PSt;
- skip -> exit_resched(infinity);
- reset -> exit_resched(reset)
+ {ok, PSt} ->
+ PSt;
+ skip ->
+ % If plugin skipped resume, count this as an attempt and
+ % reschedule for the future
+ Now = tsec(),
+ SkipReschedTSec = schedule_time(Mod, Now, Now),
+ exit_resched(SkipReschedTSec);
+ reset ->
+ exit_resched(reset)
end.
db_opened_callback(#st{pst = PSt, callbacks = Cbks, db = Db} = St) ->
diff --git a/src/couch_scanner/test/eunit/couch_scanner_test.erl b/src/couch_scanner/test/eunit/couch_scanner_test.erl
index 1fe2e157d..d288dc50a 100644
--- a/src/couch_scanner/test/eunit/couch_scanner_test.erl
+++ b/src/couch_scanner/test/eunit/couch_scanner_test.erl
@@ -29,6 +29,7 @@ couch_scanner_test_() ->
?TDEF_FE(t_conflict_finder_works, 30),
?TDEF_FE(t_config_skips, 10),
?TDEF_FE(t_resume_after_error, 10),
+ ?TDEF_FE(t_resume_after_skip, 10),
?TDEF_FE(t_reset, 10),
?TDEF_FE(t_schedule_repeat, 10),
?TDEF_FE(t_schedule_after, 15)
@@ -272,6 +273,25 @@ t_resume_after_error(_) ->
config:set("couch_scanner_plugins", Plugin, "true", false),
meck:wait(?FIND_PLUGIN, resume, 2, 10000).
+t_resume_after_skip(_) ->
+ meck:reset(?FIND_PLUGIN),
+ meck:expect(
+ ?FIND_PLUGIN,
+ start,
+ 2,
+ meck:seq([
+ skip,
+ meck:passthrough()
+ ])
+ ),
+ Plugin = atom_to_list(?FIND_PLUGIN),
+ config:set("couch_scanner", "min_penalty_sec", "1", false),
+ config:set("couch_scanner", "interval_sec", "1", false),
+ config:set(Plugin, "repeat", "2_sec", false),
+ couch_scanner:resume(),
+ config:set("couch_scanner_plugins", Plugin, "true", false),
+ meck:wait(?FIND_PLUGIN, complete, 1, 10000).
+
t_reset(_) ->
meck:reset(?FIND_PLUGIN),
meck:expect(