You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by "nickva (via GitHub)" <gi...@apache.org> on 2023/07/20 18:00:34 UTC

[GitHub] [couchdb] nickva commented on a diff in pull request #4681: Introduce optional countermeasures as we run out of disk space

nickva commented on code in PR #4681:
URL: https://github.com/apache/couchdb/pull/4681#discussion_r1269721802


##########
src/couch/src/couch_disk_monitor.erl:
##########
@@ -0,0 +1,258 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+    block_background_view_indexing/0,
+    block_interactive_view_indexing/0,
+    block_database_writes/0
+]).
+
+% testing
+-export([
+    set_database_dir_percent_used/1,
+    set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+    start_link/0,
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+    timer
+}).
+
+block_background_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled ->
+            view_index_dir_percent_used() > background_view_indexing_threshold();
+        true ->
+            false
+    end.
+
+block_interactive_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+        true -> false
+    end.
+
+block_database_writes() ->
+    Enabled = enabled(),
+    if
+        Enabled -> database_dir_percent_used() > database_writes_threshold();
+        true -> false
+    end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+    ets:new(?MODULE, [named_table]),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {ok, #st{timer = TRef}}.
+
+handle_call(_Msg, _From, St) ->
+    {reply, {error, unknown_msg}, St}.
+
+handle_cast({set_database_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {database_dir, PercentUsed}),
+    {noreply, St};
+handle_cast({set_view_index_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+    {noreply, St};
+handle_cast(_Msg, St) ->
+    {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+    timer:cancel(St#st.timer),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+    {noreply, St}.
+
+update_disk_data() ->
+    DiskData = disksup:get_disk_data(),
+    update_disk_data(DiskData).
+
+update_disk_data([]) ->
+    ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+    IsDatabaseDir = is_database_dir(Id),
+    IsViewIndexDir = is_view_index_dir(Id),
+    if
+        IsDatabaseDir ->
+            ets:insert(?MODULE, {database_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    if
+        IsViewIndexDir ->
+            ets:insert(?MODULE, {view_index_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+    same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+    same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+    case {device_id(DirA), device_id(DirB)} of
+        {{ok, DeviceId}, {ok, DeviceId}} ->
+            true;
+        _Else ->
+            false
+    end.
+
+device_id(Dir) ->
+    case file:read_file_info(Dir) of
+        {ok, FileInfo} ->
+            {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+database_dir_percent_used() ->
+    [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+    PercentUsed.
+
+view_index_dir_percent_used() ->
+    [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+    PercentUsed.
+
+background_view_indexing_threshold() ->
+    config:get_integer(?SECTION, "background_view_indexing_threshold", 80).
+
+interactive_view_indexing_threshold() ->
+    config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90).
+
+database_writes_threshold() ->
+    config:get_integer(?SECTION, "database_writes_threshold", 90).
+
+enabled() ->
+    config:get_boolean(?SECTION, "enable", false).
+
+%% Align our refresh with the os_mon refresh
+timer_interval() ->
+    application:get_env(os_mon, disk_space_check_interval, 30) * 60_000.
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+not_enabled_by_default_test() ->
+    ?assertEqual(false, enabled()),
+    ?assertEqual(false, block_background_view_indexing()),
+    ?assertEqual(false, block_interactive_view_indexing()),
+    ?assertEqual(false, block_database_writes()).
+
+setup_all() ->
+    application:ensure_all_started(os_mon),
+    {ok, CurrentDirectory} = file:get_cwd(),
+    ok = meck:new(config, [passthrough]),
+    ok = meck:expect(
+        config,
+        get,
+        fun
+            ("couchdb", "database_dir") -> CurrentDirectory;
+            ("couchdb", "view_index_dir") -> CurrentDirectory
+        end
+    ),
+    ok = meck:expect(config, get_boolean, ["disk_monitor", "enable", '_'], true),
+    ok = meck:expect(
+        config,
+        get_integer,
+        fun
+            ("disk_monitor", "background_view_indexing_threshold", _) -> 50;
+            ("disk_monitor", "interactive_view_indexing_threshold", _) -> 60;
+            ("disk_monitor", "database_writes_threshold", _) -> 70
+        end
+    ),
+    {ok, Pid} = start_link(),
+    Pid.
+
+teardown_all(Pid) ->
+    true = exit(Pid, normal),

Review Comment:
   This won't work unless the process traps exits. 
   
   If it's a gen_server perhaps [gen_server:stop/1](https://www.erlang.org/doc/man/gen_server.html#stop-1) might work here?
   
   



##########
src/couch/src/couch_disk_monitor.erl:
##########
@@ -0,0 +1,258 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+    block_background_view_indexing/0,
+    block_interactive_view_indexing/0,
+    block_database_writes/0
+]).
+
+% testing
+-export([
+    set_database_dir_percent_used/1,
+    set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+    start_link/0,
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+    timer
+}).
+
+block_background_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled ->
+            view_index_dir_percent_used() > background_view_indexing_threshold();
+        true ->
+            false
+    end.
+
+block_interactive_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+        true -> false
+    end.
+
+block_database_writes() ->
+    Enabled = enabled(),
+    if
+        Enabled -> database_dir_percent_used() > database_writes_threshold();
+        true -> false
+    end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+    ets:new(?MODULE, [named_table]),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {ok, #st{timer = TRef}}.
+
+handle_call(_Msg, _From, St) ->
+    {reply, {error, unknown_msg}, St}.
+
+handle_cast({set_database_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {database_dir, PercentUsed}),
+    {noreply, St};
+handle_cast({set_view_index_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+    {noreply, St};
+handle_cast(_Msg, St) ->
+    {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+    timer:cancel(St#st.timer),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+    {noreply, St}.
+
+update_disk_data() ->
+    DiskData = disksup:get_disk_data(),
+    update_disk_data(DiskData).
+
+update_disk_data([]) ->
+    ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+    IsDatabaseDir = is_database_dir(Id),
+    IsViewIndexDir = is_view_index_dir(Id),
+    if
+        IsDatabaseDir ->
+            ets:insert(?MODULE, {database_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    if
+        IsViewIndexDir ->
+            ets:insert(?MODULE, {view_index_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+    same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+    same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+    case {device_id(DirA), device_id(DirB)} of
+        {{ok, DeviceId}, {ok, DeviceId}} ->
+            true;
+        _Else ->
+            false
+    end.
+
+device_id(Dir) ->
+    case file:read_file_info(Dir) of
+        {ok, FileInfo} ->
+            {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+database_dir_percent_used() ->
+    [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+    PercentUsed.
+
+view_index_dir_percent_used() ->
+    [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+    PercentUsed.
+
+background_view_indexing_threshold() ->
+    config:get_integer(?SECTION, "background_view_indexing_threshold", 80).
+
+interactive_view_indexing_threshold() ->
+    config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90).
+
+database_writes_threshold() ->
+    config:get_integer(?SECTION, "database_writes_threshold", 90).
+
+enabled() ->
+    config:get_boolean(?SECTION, "enable", false).
+
+%% Align our refresh with the os_mon refresh
+timer_interval() ->
+    application:get_env(os_mon, disk_space_check_interval, 30) * 60_000.
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+not_enabled_by_default_test() ->
+    ?assertEqual(false, enabled()),
+    ?assertEqual(false, block_background_view_indexing()),
+    ?assertEqual(false, block_interactive_view_indexing()),
+    ?assertEqual(false, block_database_writes()).
+
+setup_all() ->
+    application:ensure_all_started(os_mon),

Review Comment:
   This still start both `os_mon` but also, possibly other apps like `sasl`. We have to keep track of those and make sure stop them during cleanup.
   
   A simple alternative might be to just start a full couch instance with `Ctx = test_util:start_couch()` which would start couch and its dependent apps like `os_mon` and its transitive dependencies. Then `test_util:stop_couch(Ctx)` should stop them. This would also start `config` so that might make mocking config un-necessary. To avoid config writing to files can make those update non-persistent with `config:set("section", "key", "val", _Persist=false)` in setup and in teardown remove them `config:delete("section", "key", _Persist=false)`. Removal is probably not strictly needed as we'd tear it down during cleanup but it's probably good practice 



##########
src/couch/src/couch_disk_monitor.erl:
##########
@@ -0,0 +1,258 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+    block_background_view_indexing/0,
+    block_interactive_view_indexing/0,
+    block_database_writes/0
+]).
+
+% testing
+-export([
+    set_database_dir_percent_used/1,
+    set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+    start_link/0,
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+    timer
+}).
+
+block_background_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled ->
+            view_index_dir_percent_used() > background_view_indexing_threshold();
+        true ->
+            false
+    end.
+
+block_interactive_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+        true -> false
+    end.
+
+block_database_writes() ->
+    Enabled = enabled(),
+    if
+        Enabled -> database_dir_percent_used() > database_writes_threshold();
+        true -> false
+    end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+    ets:new(?MODULE, [named_table]),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {ok, #st{timer = TRef}}.
+
+handle_call(_Msg, _From, St) ->
+    {reply, {error, unknown_msg}, St}.
+
+handle_cast({set_database_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {database_dir, PercentUsed}),
+    {noreply, St};
+handle_cast({set_view_index_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+    {noreply, St};
+handle_cast(_Msg, St) ->
+    {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+    timer:cancel(St#st.timer),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+    {noreply, St}.
+
+update_disk_data() ->
+    DiskData = disksup:get_disk_data(),
+    update_disk_data(DiskData).
+
+update_disk_data([]) ->
+    ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+    IsDatabaseDir = is_database_dir(Id),
+    IsViewIndexDir = is_view_index_dir(Id),
+    if
+        IsDatabaseDir ->
+            ets:insert(?MODULE, {database_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    if
+        IsViewIndexDir ->
+            ets:insert(?MODULE, {view_index_dir, PercentUsed});
+        true ->
+            ok
+    end,
+    update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+    same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+    same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+    case {device_id(DirA), device_id(DirB)} of
+        {{ok, DeviceId}, {ok, DeviceId}} ->
+            true;
+        _Else ->
+            false
+    end.
+
+device_id(Dir) ->
+    case file:read_file_info(Dir) of
+        {ok, FileInfo} ->
+            {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+database_dir_percent_used() ->
+    [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+    PercentUsed.
+
+view_index_dir_percent_used() ->
+    [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+    PercentUsed.

Review Comment:
   These would be in the update hot-path and could become bottlenecks for concurrent requests. Could probably add  `{read_concurrency, true} (https://www.erlang.org/doc/man/ets.html#new_2_read_concurrency) to ets creation options to match what we do with the config ets table.



##########
src/couch/src/couch_disk_monitor.erl:
##########
@@ -0,0 +1,258 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+    block_background_view_indexing/0,
+    block_interactive_view_indexing/0,
+    block_database_writes/0
+]).
+
+% testing
+-export([
+    set_database_dir_percent_used/1,
+    set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+    start_link/0,
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+    timer
+}).
+
+block_background_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled ->
+            view_index_dir_percent_used() > background_view_indexing_threshold();
+        true ->
+            false
+    end.
+
+block_interactive_view_indexing() ->
+    Enabled = enabled(),
+    if
+        Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+        true -> false
+    end.
+
+block_database_writes() ->
+    Enabled = enabled(),
+    if
+        Enabled -> database_dir_percent_used() > database_writes_threshold();
+        true -> false
+    end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+    gen_server:cast(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+    ets:new(?MODULE, [named_table]),
+    update_disk_data(),
+    {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+    {ok, #st{timer = TRef}}.
+
+handle_call(_Msg, _From, St) ->
+    {reply, {error, unknown_msg}, St}.
+
+handle_cast({set_database_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {database_dir, PercentUsed}),
+    {noreply, St};
+handle_cast({set_view_index_dir_percent_used, PercentUsed}, St) ->
+    ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+    {noreply, St};
+handle_cast(_Msg, St) ->
+    {noreply, St}.

Review Comment:
   Usually in most gen_servers in couch we stop with an error we if get an un-expected cast or info.



##########
src/couch_mrview/src/couch_mrview_util.erl:
##########
@@ -190,12 +191,24 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
                     couch_index:get_state(Pid, 0);
                 false ->
                     couch_index:get_state(Pid, 0);
+                _ when BlockInteractiveIndexing ->
+                    %% if the view is already fresh enough, proceed.
+                    %% otherwise it's an error until there's room to update.
+                    case couch_index:get_state(Pid, 0) of

Review Comment:
   Should we block "lazy" building a as well? Would the lazy update be prevented in at the cluster (fabric) level in `maybe_update_others(DbName,DDoc,ShardsInvolved,ViewName, #mrargs{update = lazy} = Args) ->`? The one issue in that case is the coordinator might not know what the disk usage would be on the worker node.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: notifications-unsubscribe@couchdb.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org