You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ji...@apache.org on 2020/04/21 06:05:38 UTC

[couchdb-couch-dbupdates] 01/01: background deletion for soft-deleted database

This is an automated email from the ASF dual-hosted git repository.

jiangphcn pushed a commit to branch background-delete
in repository https://gitbox.apache.org/repos/asf/couchdb-couch-dbupdates.git

commit f7435fbba8791c9d146bfa59353de6b66a1d2727
Author: jiangph <ji...@cn.ibm.com>
AuthorDate: Tue Apr 21 14:02:16 2020 +0800

    background deletion for soft-deleted database
    
    allow background job to delete soft-deleted database according to
    specified criteria to release space. Once database is hard-deleted,
    the data can't be fetched back.
---
 include/couch_dbupdates.hrl   |  21 ++++++
 src/couch_dbdelete_app.erl    |  31 ++++++++
 src/couch_dbdelete_server.erl | 165 ++++++++++++++++++++++++++++++++++++++++++
 src/couch_dbdelete_sup.erl    |  61 ++++++++++++++++
 src/couch_dbdelete_worker.erl |  51 +++++++++++++
 src/couch_dbupdates.app.src   |  14 +++-
 6 files changed, 341 insertions(+), 2 deletions(-)

diff --git a/include/couch_dbupdates.hrl b/include/couch_dbupdates.hrl
new file mode 100644
index 0000000..7bb3557
--- /dev/null
+++ b/include/couch_dbupdates.hrl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% jobs api
+-define(DB_DELETE_JOB_TYPE, <<"dbdelete">>).
+
+% number of worker for db background deletion
+-define(MAX_WORKERS, 10).
+
+% settings for background deletion interval
+-define(DEFAULT_DELAY_MSEC, 60000).
+-define(DEFAULT_RESOLUTION_MSEC, 10000).
\ No newline at end of file
diff --git a/src/couch_dbdelete_app.erl b/src/couch_dbdelete_app.erl
new file mode 100644
index 0000000..757b0af
--- /dev/null
+++ b/src/couch_dbdelete_app.erl
@@ -0,0 +1,31 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_dbdelete_app).
+
+
+-behaviour(application).
+
+
+-export([
+    start/2,
+    stop/1
+]).
+
+
+start(_StartType, _StartArgs) ->
+    couch_dbdelete_sup:start_link().
+
+
+stop(_State) ->
+    ok.
diff --git a/src/couch_dbdelete_server.erl b/src/couch_dbdelete_server.erl
new file mode 100644
index 0000000..7c78961
--- /dev/null
+++ b/src/couch_dbdelete_server.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_dbdelete_server).
+
+
+-behaviour(gen_server).
+
+
+-export([
+    start_link/0,
+    process_loop/0
+]).
+
+
+-export([
+    init/1,
+    terminate/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    code_change/3
+]).
+
+
+-include("couch_dbupdates.hrl").
+
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+    process_flag(trap_exit, true),
+    proc_lib:spawn_link(?MODULE, process_loop, []),
+    couch_jobs:set_type_timeout(?DB_DELETE_JOB_TYPE, 6),
+    St = #{
+        workers => #{},
+        max_workers => max_workers()
+    },
+    {ok, spawn_workers(St)}.
+
+
+terminate(_, _St) ->
+    ok.
+
+
+handle_call(Msg, _From, St) ->
+    {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+    {stop, {bad_cast, Msg}, St}.
+
+
+handle_info({'EXIT', Pid, Reason}, St) ->
+    #{workers := Workers} = St,
+    case maps:is_key(Pid, Workers) of
+        true ->
+            if Reason == normal -> ok; true ->
+                LogMsg = "~p : indexer process ~p exited with ~p",
+                couch_log:error(LogMsg, [?MODULE, Pid, Reason])
+            end,
+            NewWorkers = maps:remove(Pid, Workers),
+            {noreply, spawn_workers(St#{workers := NewWorkers})};
+        false ->
+            LogMsg = "~p : unknown process ~p exited with ~p",
+            couch_log:error(LogMsg, [?MODULE, Pid, Reason]),
+            {stop, {unknown_pid_exit, Pid}, St}
+    end;
+
+handle_info(Msg, St) ->
+    {stop, {bad_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+    {ok, St}.
+
+
+process_loop() ->
+    Now = now_msec(),
+    Delay = delay_msec(),
+    Since = Now - Delay,
+    case is_enabled() of
+        true ->
+            process_delete();
+        false ->
+            ok
+    end,
+    Resolution = resolution_msec(),
+    Jitter = rand:uniform(1 + Resolution div 2),
+    timer:sleep(Resolution + Jitter),
+    couch_log:error("33after sleep Resolution ~p", [Resolution]),
+    couch_log:error("333after sleep Jitter ~p", [Jitter]),
+    process_loop().
+
+
+process_delete() ->
+    {ok, Infos} = fabric2_db:list_deleted_dbs_info(),
+    couch_log:error("process_delete ~p", [Infos]),
+
+    lists:foreach(fun(Info) ->
+        DbName = proplists:get_value(db_name, Info),
+        Timestamp = proplists:get_value(timestamp, Info),
+        JobId = job_id(DbName, Timestamp),
+        JobData = job_data(DbName, Timestamp),
+        ok = couch_jobs:add(undefined, ?DB_DELETE_JOB_TYPE, JobId, JobData)
+                  end, Infos).
+
+
+job_id(DbName, Timestamp) ->
+    <<DbName/binary, "-", Timestamp/binary>>.
+
+
+job_data(DbName, Timestamp) ->
+    #{
+        db_name => DbName,
+        timestamp => Timestamp
+    }.
+
+
+now_msec() ->
+    erlang:monotonic_time(millisecond).
+
+
+is_enabled() ->
+    config:get_boolean("fabric", "db_delete_enabled", true).
+
+
+delay_msec() ->
+    config:get_integer("fabric", "db_delete_delay_msec",
+        ?DEFAULT_DELAY_MSEC).
+
+
+resolution_msec() ->
+    config:get_integer("fabric", "db_delete_resolution_msec",
+        ?DEFAULT_RESOLUTION_MSEC).
+
+
+spawn_workers(St) ->
+    #{
+        workers := Workers,
+        max_workers := MaxWorkers
+    } = St,
+    case maps:size(Workers) < MaxWorkers of
+        true ->
+            Pid = couch_dbdelete_worker:spawn_link(),
+            NewSt = St#{workers := Workers#{Pid => true}},
+            spawn_workers(NewSt);
+        false ->
+            St
+    end.
+
+
+max_workers() ->
+    config:get_integer("couch_views", "max_workers", ?MAX_WORKERS).
diff --git a/src/couch_dbdelete_sup.erl b/src/couch_dbdelete_sup.erl
new file mode 100644
index 0000000..70d7a95
--- /dev/null
+++ b/src/couch_dbdelete_sup.erl
@@ -0,0 +1,61 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_dbdelete_sup).
+
+
+-behaviour(supervisor).
+
+
+-export([
+    start_link/0
+]).
+
+
+-export([
+    init/1
+]).
+
+
+-include("couch_dbupdates.hrl").
+
+
+start_link() ->
+    Arg = case fabric2_node_types:is_type(db_bg_deletion) of
+          true -> normal;
+          false -> builds_disabled
+        end,
+    supervisor:start_link({local, ?MODULE}, ?MODULE, Arg).
+
+
+init(normal) ->
+    Children = [
+        #{
+            id => couch_dbdelete_server,
+            start => {couch_dbdelete_server, start_link, []}
+        }
+    ],
+    {ok, {flags(), Children}};
+
+init(builds_disabled) ->
+    couch_log:notice("~p : db background deletion disabled", [?MODULE]),
+    couch_jobs:set_type_timeout(?DB_DELETE_JOB_TYPE, 6),
+    {ok, {flags(), []}}.
+
+
+flags() ->
+    #{
+        strategy => one_for_one,
+        intensity => 1,
+        period => 5
+    }.
diff --git a/src/couch_dbdelete_worker.erl b/src/couch_dbdelete_worker.erl
new file mode 100644
index 0000000..88fe336
--- /dev/null
+++ b/src/couch_dbdelete_worker.erl
@@ -0,0 +1,51 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_dbdelete_worker).
+
+-export([
+    spawn_link/0
+]).
+
+
+-export([
+    init/0
+]).
+
+-ifdef(TEST).
+-compile(export_all).
+-compile(nowarn_export_all).
+-endif.
+
+-include("couch_dbupdates.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+
+
+spawn_link() ->
+    proc_lib:spawn_link(?MODULE, init, []).
+
+
+init() ->
+    {ok, Job, JobData} = couch_jobs:accept(?DB_DELETE_JOB_TYPE, #{}),
+    #{
+        <<"db_name">> := DbName,
+        <<"timestamp">> := TimeStamp
+    } = JobData,
+
+    ok = fabric2_db:delete(DbName, [{deleted_at, TimeStamp}]),
+
+    couch_jobs:finish(undefined, Job, JobData#{
+        message => db_deleted,
+        reason => "Database was deleted"
+    }),
+    exit(normal).
diff --git a/src/couch_dbupdates.app.src b/src/couch_dbupdates.app.src
index a5597ba..48819be 100644
--- a/src/couch_dbupdates.app.src
+++ b/src/couch_dbupdates.app.src
@@ -2,10 +2,20 @@
  [
   {description, ""},
   {vsn, git},
-  {registered, []},
+  {mod, {couch_dbdelete_app, []}},
+  {registered,  [
+                 couch_dbdelete_sup,
+                 couch_dbdelete_server
+                ]},
   {applications, [
                   kernel,
-                  stdlib
+                  stdlib,
+                  erlfdb,
+                  couch_epi,
+                  couch_log,
+                  config,
+                  fabric,
+                  couch_jobs
                  ]},
   {env, []}
  ]}.