You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2025/09/15 11:08:43 UTC
(couchdb) branch auto-delete-tseq updated (fe71ac083 -> bca189f99)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch auto-delete-tseq
in repository https://gitbox.apache.org/repos/asf/couchdb.git
discard fe71ac083 purge deleted documents that exceed TTL
new bca189f99 purge deleted documents that exceed TTL
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fe71ac083)
\
N -- N -- N refs/heads/auto-delete-tseq (bca189f99)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/src/couch_auto_purge_plugin.erl | 11 +++++---
.../test/eunit/couch_auto_purge_plugin_tests.erl | 32 ++++++++++++++++++----
2 files changed, 33 insertions(+), 10 deletions(-)
(couchdb) 01/01: purge deleted documents that exceed TTL
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch auto-delete-tseq
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bca189f99ad2d9da41d778bbf2c1b9dd6112a6a0
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Sep 2 15:32:21 2025 +0100
purge deleted documents that exceed TTL
---
src/couch/src/couch_auto_purge_plugin.erl | 169 +++++++++++++++++++++
.../test/eunit/couch_auto_purge_plugin_tests.erl | 118 ++++++++++++++
src/docs/src/config/scanner.rst | 9 ++
3 files changed, 296 insertions(+)
diff --git a/src/couch/src/couch_auto_purge_plugin.erl b/src/couch/src/couch_auto_purge_plugin.erl
new file mode 100644
index 000000000..c6f8b885c
--- /dev/null
+++ b/src/couch/src/couch_auto_purge_plugin.erl
@@ -0,0 +1,169 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_auto_purge_plugin).
+-behaviour(couch_scanner_plugin).
+
+-export([
+ start/2,
+ resume/2,
+ complete/1,
+ checkpoint/1,
+ db/2,
+ db_opened/2,
+ db_closing/2,
+ doc_fdi/3
+]).
+
+-include_lib("couch_scanner/include/couch_scanner_plugin.hrl").
+-include_lib("stdlib/include/assert.hrl").
+
+start(ScanId, #{}) ->
+ St = init_config(ScanId),
+ ?INFO("Starting.", [], St),
+ {ok, St}.
+
+resume(ScanId, #{}) ->
+ St = init_config(ScanId),
+ ?INFO("Resuming.", [], St),
+ {ok, St}.
+
+complete(St) ->
+ ?INFO("Completed", [], St),
+ {ok, #{}}.
+
+checkpoint(_St) ->
+ {ok, #{}}.
+
+db(St, DbName) ->
+ case ttl(St, DbName) of
+ TTL when is_integer(TTL) ->
+ {ok, St#{ttl => TTL}};
+ undefined ->
+ {skip, St}
+ end.
+
+db_opened(#{} = St, Db) ->
+ #{ttl := TTL} = St,
+ EndSeq = couch_time_seq:since(couch_db:get_time_seq(Db), couch_time_seq:timestamp() - TTL),
+ ChangeOpts =
+ if
+ EndSeq == now -> [];
+ true -> [{end_key, EndSeq}]
+ end,
+ ?INFO("scanning for deleted documents in ~s up to ~p", [couch_db:name(Db), EndSeq], meta(St)),
+ {0, ChangeOpts, St#{count => 0, end_seq => EndSeq}}.
+
+db_closing(#{} = St, Db) ->
+ #{count := Count} = St,
+ ?INFO("purged ~B deleted documents from ~s", [Count, couch_db:name(Db)], meta(St)),
+ {ok, St}.
+
+doc_fdi(#{} = St, #full_doc_info{deleted = true} = FDI, Db) ->
+ #{end_seq := EndSeq} = St,
+ ?assert(
+ FDI#full_doc_info.update_seq =< EndSeq, "FDI update_seq should not be greater than end seq"
+ ),
+ {ok, purge(St, FDI, Db)};
+doc_fdi(#{} = St, #full_doc_info{}, _Db) ->
+ {ok, St}.
+
+purge(#{} = St, #full_doc_info{} = FDI, Db) ->
+ {Id, Revs} = fdi_to_idrevs(FDI),
+ MaxBatchSize = config:get_integer(atom_to_list(?MODULE), "max_batch_size", 500),
+ purge(St, Id, Revs, MaxBatchSize, Db).
+
+purge(#{} = St, Id, Revs, MaxBatchSize, Db) when length(Revs) =< MaxBatchSize ->
+ DbName = mem3:dbname(couch_db:name(Db)),
+ PurgeFun = fun() -> fabric:purge_docs(DbName, [{Id, Revs}], [?ADMIN_CTX]) end,
+ Timeout = fabric_util:request_timeout(),
+ try fabric_util:isolate(PurgeFun, Timeout) of
+ {Health, Results} when Health == ok; Health == accepted ->
+ #{count := Count, limiter := Limiter0} = St,
+ {Wait, Limiter1} = couch_scanner_rate_limiter:update(
+ Limiter0, doc_write, length(Results)
+ ),
+ timer:sleep(Wait),
+ St#{count => Count + length(Results), limiter => Limiter1};
+ Else ->
+ ?WARN(
+ "Failed to purge deleted documents in ~s/~s for reason ~p",
+ [DbName, Id, Else],
+ meta(St)
+ ),
+ St
+ catch
+ Class:Reason ->
+ ?WARN(
+ "Failed to purge deleted documents in ~s/~s for reason ~p:~p",
+ [DbName, Id, Class, Reason],
+ meta(St)
+ ),
+ St
+ end;
+purge(#{} = St0, Id, Revs, MaxBatchSize, Db) ->
+ {RevBatch, RevRest} = lists:split(MaxBatchSize, Revs),
+ St1 = purge(St0, Id, RevBatch, MaxBatchSize, Db),
+ purge(St1, Id, RevRest, MaxBatchSize, Db).
+
+fdi_to_idrevs(#full_doc_info{} = FDI) ->
+ Revs = [
+ couch_doc:rev_to_str({Pos, RevId})
+ || {#leaf{}, {Pos, [RevId | _]}} <- couch_key_tree:get_all_leafs(FDI#full_doc_info.rev_tree)
+ ],
+ {FDI#full_doc_info.id, Revs}.
+
+init_config(ScanId) ->
+ #{sid => ScanId, limiter => couch_scanner_rate_limiter:get()}.
+
+meta(#{sid := ScanId}) ->
+ #{sid => ScanId}.
+
+ttl(St, DbName) ->
+ DefaultTTL = config:get(atom_to_list(?MODULE), "deleted_document_ttl"),
+ GetFun = fun() -> fabric:open_doc(DbName, <<"_local/_auto_purge">>, [?ADMIN_CTX]) end,
+ Timeout = fabric_util:request_timeout(),
+ DbTTL =
+ try fabric_util:isolate(GetFun, Timeout) of
+ {ok, #doc{} = Doc} ->
+ {Props} = couch_doc:to_json_obj(Doc, []),
+ case couch_util:get_value(<<"deleted_document_ttl">>, Props) of
+ TTL when is_integer(TTL) ->
+ TTL;
+ Else ->
+ ?WARN(
+ "TTL in ~s as ttl was '~p', not integer",
+ [DbName, Else],
+ meta(St)
+ ),
+ undefined
+ end;
+ {not_found, _Reason} ->
+ undefined;
+ {error, Reason} ->
+ ?WARN(
+ "Failed to fetch ttl in ~s for reason ~p",
+ [DbName, Reason],
+ meta(St)
+ ),
+ undefined
+ catch
+ Class:Reason ->
+ ?WARN(
+ "Failed to fetch ttl in ~s for reason ~p:~p", [DbName, Class, Reason], meta(St)
+ ),
+ undefined
+ end,
+ if
+ DbTTL /= undefined -> DbTTL;
+ true -> list_to_integer(DefaultTTL)
+ end.
diff --git a/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl b/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl
new file mode 100644
index 000000000..2e9bbfe8d
--- /dev/null
+++ b/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl
@@ -0,0 +1,118 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_auto_purge_plugin_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(PLUGIN, couch_auto_purge_plugin).
+
+couch_quickjs_scanner_plugin_test_() ->
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(t_no_auto_purge_by_default, 10),
+ ?TDEF_FE(t_auto_purge_after_config_ttl, 10),
+ ?TDEF_FE(t_auto_purge_after_db_ttl, 10)
+ ]
+ }.
+
+setup() ->
+ {module, _} = code:ensure_loaded(?PLUGIN),
+ meck:new(?PLUGIN, [passthrough]),
+ meck:new(couch_scanner_server, [passthrough]),
+ meck:new(couch_scanner_util, [passthrough]),
+ Ctx = test_util:start_couch([fabric, couch_scanner]),
+ DbName = ?tempdb(),
+ ok = fabric:create_db(DbName, [{q, "2"}, {n, "1"}]),
+ config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false),
+ reset_stats(),
+ {Ctx, DbName}.
+
+teardown({Ctx, DbName}) ->
+ config_delete_section("couch_scanner"),
+ config_delete_section("couch_scanner_plugins"),
+ config_delete_section(atom_to_list(?PLUGIN)),
+ couch_scanner:reset_checkpoints(),
+ couch_scanner:resume(),
+ fabric:delete_db(DbName),
+ test_util:stop_couch(Ctx),
+ meck:unload().
+
+t_no_auto_purge_by_default({_, DbName}) ->
+ ok = add_doc(DbName, <<"doc1">>, #{<<"_deleted">> => true}),
+ ?assertEqual(1, doc_del_count(DbName)),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(1, doc_del_count(DbName)),
+ ok.
+
+t_auto_purge_after_config_ttl({_, DbName}) ->
+ config:set(atom_to_list(?PLUGIN), "deleted_document_ttl", "-1000000", false),
+ ok = add_doc(DbName, <<"doc1">>, #{<<"_deleted">> => true}),
+ ?assertEqual(1, doc_del_count(DbName)),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(0, doc_del_count(DbName)),
+ ok.
+
+t_auto_purge_after_db_ttl({_, DbName}) ->
+ ok = add_doc(DbName, <<"_local/_auto_purge">>, #{<<"deleted_document_ttl">> => -1_000_000}),
+ ok = add_doc(DbName, <<"doc1">>, #{<<"_deleted">> => true}),
+ ?assertEqual(1, doc_del_count(DbName)),
+ meck:reset(couch_scanner_server),
+ meck:reset(?PLUGIN),
+ config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false),
+ wait_exit(10000),
+ ?assertEqual(0, doc_del_count(DbName)),
+ ok.
+
+reset_stats() ->
+ Counters = [
+ [couchdb, query_server, process_error_exits],
+ [couchdb, query_server, process_errors],
+ [couchdb, query_server, process_exits]
+ ],
+ [reset_counter(C) || C <- Counters].
+
+reset_counter(Counter) ->
+ case couch_stats:sample(Counter) of
+ 0 ->
+ ok;
+ N when is_integer(N), N > 0 ->
+ couch_stats:decrement_counter(Counter, N)
+ end.
+
+config_delete_section(Section) ->
+ [config:delete(K, V, false) || {K, V} <- config:get(Section)].
+
+add_doc(DbName, DocId, Body) ->
+ {ok, _} = fabric:update_doc(DbName, mkdoc(DocId, Body), [?ADMIN_CTX]),
+ ok.
+
+mkdoc(Id, #{} = Body) ->
+ Body1 = Body#{<<"_id">> => Id},
+ jiffy:decode(jiffy:encode(Body1)).
+
+wait_exit(MSec) ->
+ meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'], MSec).
+
+doc_del_count(DbName) ->
+ {ok, DbInfo} = fabric:get_db_info(DbName),
+ couch_util:get_value(doc_del_count, DbInfo).
diff --git a/src/docs/src/config/scanner.rst b/src/docs/src/config/scanner.rst
index f36619f4a..fc4ed5a97 100644
--- a/src/docs/src/config/scanner.rst
+++ b/src/docs/src/config/scanner.rst
@@ -249,3 +249,12 @@ settings in their ``[{plugin}]`` section.
[couch_scanner_plugin_ddoc_features]
ddoc_report = false
+
+.. config:section:: couch_auto_purge_plugin :: Configure the Auto Purge plugin
+
+ .. config:option:: deleted_document_ttl
+
+ Set the default interval, in seconds, before the plugin will purge
+ a deleted document. The database may override this setting with the
+ ``/dbname/_deleted_document_ttl`` endpoint. If neither is set, the
+ plugin will not purge deleted documents.