You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ji...@apache.org on 2020/04/21 06:05:38 UTC
[couchdb-couch-dbupdates] 01/01: background deletion for
soft-deleted database
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch background-delete
in repository https://gitbox.apache.org/repos/asf/couchdb-couch-dbupdates.git
commit f7435fbba8791c9d146bfa59353de6b66a1d2727
Author: jiangph <ji...@cn.ibm.com>
AuthorDate: Tue Apr 21 14:02:16 2020 +0800
background deletion for soft-deleted database
allow background job to delete soft-deleted database according to
specified criteria to release space. Once database is hard-deleted,
the data can't be fetched back.
---
include/couch_dbupdates.hrl | 21 ++++++
src/couch_dbdelete_app.erl | 31 ++++++++
src/couch_dbdelete_server.erl | 165 ++++++++++++++++++++++++++++++++++++++++++
src/couch_dbdelete_sup.erl | 61 ++++++++++++++++
src/couch_dbdelete_worker.erl | 51 +++++++++++++
src/couch_dbupdates.app.src | 14 +++-
6 files changed, 341 insertions(+), 2 deletions(-)
diff --git a/include/couch_dbupdates.hrl b/include/couch_dbupdates.hrl
new file mode 100644
index 0000000..7bb3557
--- /dev/null
+++ b/include/couch_dbupdates.hrl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% jobs api
+-define(DB_DELETE_JOB_TYPE, <<"dbdelete">>).
+
+% number of worker for db background deletion
+-define(MAX_WORKERS, 10).
+
+% settings for background deletion interval
+-define(DEFAULT_DELAY_MSEC, 60000).
+-define(DEFAULT_RESOLUTION_MSEC, 10000).
\ No newline at end of file
diff --git a/src/couch_dbdelete_app.erl b/src/couch_dbdelete_app.erl
new file mode 100644
index 0000000..757b0af
--- /dev/null
+++ b/src/couch_dbdelete_app.erl
@@ -0,0 +1,31 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_dbdelete_app).
+
+
+-behaviour(application).
+
+
+-export([
+ start/2,
+ stop/1
+]).
+
+
+start(_StartType, _StartArgs) ->
+ couch_dbdelete_sup:start_link().
+
+
+stop(_State) ->
+ ok.
diff --git a/src/couch_dbdelete_server.erl b/src/couch_dbdelete_server.erl
new file mode 100644
index 0000000..7c78961
--- /dev/null
+++ b/src/couch_dbdelete_server.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_dbdelete_server).
+
+
+-behaviour(gen_server).
+
+
+-export([
+ start_link/0,
+ process_loop/0
+]).
+
+
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+-include("couch_dbupdates.hrl").
+
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+ process_flag(trap_exit, true),
+ proc_lib:spawn_link(?MODULE, process_loop, []),
+ couch_jobs:set_type_timeout(?DB_DELETE_JOB_TYPE, 6),
+ St = #{
+ workers => #{},
+ max_workers => max_workers()
+ },
+ {ok, spawn_workers(St)}.
+
+
+terminate(_, _St) ->
+ ok.
+
+
+handle_call(Msg, _From, St) ->
+ {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+ {stop, {bad_cast, Msg}, St}.
+
+
+handle_info({'EXIT', Pid, Reason}, St) ->
+ #{workers := Workers} = St,
+ case maps:is_key(Pid, Workers) of
+ true ->
+ if Reason == normal -> ok; true ->
+ LogMsg = "~p : indexer process ~p exited with ~p",
+ couch_log:error(LogMsg, [?MODULE, Pid, Reason])
+ end,
+ NewWorkers = maps:remove(Pid, Workers),
+ {noreply, spawn_workers(St#{workers := NewWorkers})};
+ false ->
+ LogMsg = "~p : unknown process ~p exited with ~p",
+ couch_log:error(LogMsg, [?MODULE, Pid, Reason]),
+ {stop, {unknown_pid_exit, Pid}, St}
+ end;
+
+handle_info(Msg, St) ->
+ {stop, {bad_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+process_loop() ->
+ Now = now_msec(),
+ Delay = delay_msec(),
+ Since = Now - Delay,
+ case is_enabled() of
+ true ->
+ process_delete();
+ false ->
+ ok
+ end,
+ Resolution = resolution_msec(),
+ Jitter = rand:uniform(1 + Resolution div 2),
+ timer:sleep(Resolution + Jitter),
+ couch_log:error("33after sleep Resolution ~p", [Resolution]),
+ couch_log:error("333after sleep Jitter ~p", [Jitter]),
+ process_loop().
+
+
+process_delete() ->
+ {ok, Infos} = fabric2_db:list_deleted_dbs_info(),
+ couch_log:error("process_delete ~p", [Infos]),
+
+ lists:foreach(fun(Info) ->
+ DbName = proplists:get_value(db_name, Info),
+ Timestamp = proplists:get_value(timestamp, Info),
+ JobId = job_id(DbName, Timestamp),
+ JobData = job_data(DbName, Timestamp),
+ ok = couch_jobs:add(undefined, ?DB_DELETE_JOB_TYPE, JobId, JobData)
+ end, Infos).
+
+
+job_id(DbName, Timestamp) ->
+ <<DbName/binary, "-", Timestamp/binary>>.
+
+
+job_data(DbName, Timestamp) ->
+ #{
+ db_name => DbName,
+ timestamp => Timestamp
+ }.
+
+
+now_msec() ->
+ erlang:monotonic_time(millisecond).
+
+
+is_enabled() ->
+ config:get_boolean("fabric", "db_delete_enabled", true).
+
+
+delay_msec() ->
+ config:get_integer("fabric", "db_delete_delay_msec",
+ ?DEFAULT_DELAY_MSEC).
+
+
+resolution_msec() ->
+ config:get_integer("fabric", "db_delete_resolution_msec",
+ ?DEFAULT_RESOLUTION_MSEC).
+
+
+spawn_workers(St) ->
+ #{
+ workers := Workers,
+ max_workers := MaxWorkers
+ } = St,
+ case maps:size(Workers) < MaxWorkers of
+ true ->
+ Pid = couch_dbdelete_worker:spawn_link(),
+ NewSt = St#{workers := Workers#{Pid => true}},
+ spawn_workers(NewSt);
+ false ->
+ St
+ end.
+
+
+max_workers() ->
+ config:get_integer("couch_views", "max_workers", ?MAX_WORKERS).
diff --git a/src/couch_dbdelete_sup.erl b/src/couch_dbdelete_sup.erl
new file mode 100644
index 0000000..70d7a95
--- /dev/null
+++ b/src/couch_dbdelete_sup.erl
@@ -0,0 +1,61 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_dbdelete_sup).
+
+
+-behaviour(supervisor).
+
+
+-export([
+ start_link/0
+]).
+
+
+-export([
+ init/1
+]).
+
+
+-include("couch_dbupdates.hrl").
+
+
+start_link() ->
+ Arg = case fabric2_node_types:is_type(db_bg_deletion) of
+ true -> normal;
+ false -> builds_disabled
+ end,
+ supervisor:start_link({local, ?MODULE}, ?MODULE, Arg).
+
+
+init(normal) ->
+ Children = [
+ #{
+ id => couch_dbdelete_server,
+ start => {couch_dbdelete_server, start_link, []}
+ }
+ ],
+ {ok, {flags(), Children}};
+
+init(builds_disabled) ->
+ couch_log:notice("~p : db background deletion disabled", [?MODULE]),
+ couch_jobs:set_type_timeout(?DB_DELETE_JOB_TYPE, 6),
+ {ok, {flags(), []}}.
+
+
+flags() ->
+ #{
+ strategy => one_for_one,
+ intensity => 1,
+ period => 5
+ }.
diff --git a/src/couch_dbdelete_worker.erl b/src/couch_dbdelete_worker.erl
new file mode 100644
index 0000000..88fe336
--- /dev/null
+++ b/src/couch_dbdelete_worker.erl
@@ -0,0 +1,51 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_dbdelete_worker).
+
+-export([
+ spawn_link/0
+]).
+
+
+-export([
+ init/0
+]).
+
+-ifdef(TEST).
+-compile(export_all).
+-compile(nowarn_export_all).
+-endif.
+
+-include("couch_dbupdates.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+
+
+spawn_link() ->
+ proc_lib:spawn_link(?MODULE, init, []).
+
+
+init() ->
+ {ok, Job, JobData} = couch_jobs:accept(?DB_DELETE_JOB_TYPE, #{}),
+ #{
+ <<"db_name">> := DbName,
+ <<"timestamp">> := TimeStamp
+ } = JobData,
+
+ ok = fabric2_db:delete(DbName, [{deleted_at, TimeStamp}]),
+
+ couch_jobs:finish(undefined, Job, JobData#{
+ message => db_deleted,
+ reason => "Database was deleted"
+ }),
+ exit(normal).
diff --git a/src/couch_dbupdates.app.src b/src/couch_dbupdates.app.src
index a5597ba..48819be 100644
--- a/src/couch_dbupdates.app.src
+++ b/src/couch_dbupdates.app.src
@@ -2,10 +2,20 @@
[
{description, ""},
{vsn, git},
- {registered, []},
+ {mod, {couch_dbdelete_app, []}},
+ {registered, [
+ couch_dbdelete_sup,
+ couch_dbdelete_server
+ ]},
{applications, [
kernel,
- stdlib
+ stdlib,
+ erlfdb,
+ couch_epi,
+ couch_log,
+ config,
+ fabric,
+ couch_jobs
]},
{env, []}
]}.