You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2023/07/31 21:40:53 UTC
[couchdb] branch out-of-disk-handler updated (f5379dc15 -> bc0dd10bd)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch out-of-disk-handler
in repository https://gitbox.apache.org/repos/asf/couchdb.git
omit f5379dc15 Introduce optional countermeasures as we run out of disk space
new bc0dd10bd Introduce optional countermeasures as we run out of disk space
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f5379dc15)
\
N -- N -- N refs/heads/out-of-disk-handler (bc0dd10bd)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rel/overlay/etc/vm.args | 3 +++
src/couch/src/couch_disk_monitor.erl | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
[couchdb] 01/01: Introduce optional countermeasures as we run out of disk space
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch out-of-disk-handler
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bc0dd10bdb1e1fd91e0e52b3327f97a484b1a3de
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Jun 16 09:25:41 2023 +0100
Introduce optional countermeasures as we run out of disk space
---
rel/overlay/etc/default.ini | 6 +
rel/overlay/etc/vm.args | 12 ++
rel/reltool.config | 2 +
src/chttpd/src/chttpd.erl | 2 +
src/couch/priv/stats_descriptions.cfg | 4 +
src/couch/src/couch.app.src | 1 +
src/couch/src/couch_db.erl | 9 ++
src/couch/src/couch_disk_monitor.erl | 241 +++++++++++++++++++++++++++++
src/couch/src/couch_secondary_sup.erl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 8 +-
src/couch_mrview/src/couch_mrview_util.erl | 17 +-
src/docs/src/config/disk-monitor.rst | 76 +++++++++
src/docs/src/config/index.rst | 1 +
src/fabric/src/fabric_rpc.erl | 8 +-
src/fabric/src/fabric_streams.erl | 4 +-
src/fabric/src/fabric_view.erl | 9 +-
src/fabric/src/fabric_view_map.erl | 6 +
src/fabric/src/fabric_view_reduce.erl | 4 +
src/ken/src/ken_server.erl | 9 +-
test/elixir/test/config/suite.elixir | 5 +
test/elixir/test/disk_monitor.exs | 55 +++++++
21 files changed, 469 insertions(+), 13 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index bb440e0aa..c3124a643 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -898,3 +898,9 @@ port = {{prometheus_port}}
[nouveau]
enable = {{with_nouveau}}
+
+[disk_monitor]
+;enable = false
+;background_view_indexing_threshold = 80
+;interactive_view_indexing_threshold = 90
+;interactive_database_writes_threshold = 90
diff --git a/rel/overlay/etc/vm.args b/rel/overlay/etc/vm.args
index 886bbb903..5143477d6 100644
--- a/rel/overlay/etc/vm.args
+++ b/rel/overlay/etc/vm.args
@@ -122,3 +122,15 @@
# in the features list.
#
#-crypto fips_mode true
+
+# OS Mon Settings
+
+# only start disksup
+-os_mon start_cpu_sup false
+-os_mon start_memsup false
+
+# Check disk space every 5 minutes
+-os_mon disk_space_check_interval 5
+
+# don't let disksup send alerts
+-os_mon disk_almost_full_threshold 100.0
diff --git a/rel/reltool.config b/rel/reltool.config
index da9ad6d3b..1dca6ff85 100644
--- a/rel/reltool.config
+++ b/rel/reltool.config
@@ -23,6 +23,7 @@
sasl,
ssl,
stdlib,
+ os_mon,
syntax_tools,
xmerl,
%% couchdb
@@ -84,6 +85,7 @@
{app, sasl, [{incl_cond, include}]},
{app, ssl, [{incl_cond, include}]},
{app, stdlib, [{incl_cond, include}]},
+ {app, os_mon, [{incl_cond, include}]},
{app, syntax_tools, [{incl_cond, include}]},
{app, xmerl, [{incl_cond, include}]},
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 53abc731f..c8e6fdc97 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1138,6 +1138,8 @@ error_info(timeout) ->
>>};
error_info({service_unavailable, Reason}) ->
{503, <<"service unavailable">>, Reason};
+error_info({insufficient_storage, Reason}) ->
+ {507, <<"insufficent_storage">>, Reason};
error_info({timeout, _Reason}) ->
error_info(timeout);
error_info({'EXIT', {Error, _Stack}}) ->
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index 6c0d4dad2..1983eed9b 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -266,6 +266,10 @@
{type, counter},
{desc, <<"number of HTTP 503 Service unavailable responses">>}
]}.
+{[couchdb, httpd_status_codes, 507], [
+ {type, counter},
+ {desc, <<"number of HTTP 507 Insufficient Storage responses">>}
+]}.
{[couchdb, open_databases], [
{type, counter},
{desc, <<"number of open databases">>}
diff --git a/src/couch/src/couch.app.src b/src/couch/src/couch.app.src
index ef4e5e956..6fc293ac1 100644
--- a/src/couch/src/couch.app.src
+++ b/src/couch/src/couch.app.src
@@ -33,6 +33,7 @@
sasl,
inets,
ssl,
+ os_mon,
% Upstream deps
ibrowse,
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 4bc2aca8f..2ef89ced3 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -1338,6 +1338,15 @@ update_docs(Db, Docs0, Options, ?REPLICATED_CHANGES) ->
),
{ok, DocErrors};
update_docs(Db, Docs0, Options, ?INTERACTIVE_EDIT) ->
+ BlockInteractiveDatabaseWrites = couch_disk_monitor:block_interactive_database_writes(),
+ if
+ BlockInteractiveDatabaseWrites ->
+ {ok, [{insufficient_storage, <<"database_dir is too full">>} || _ <- Docs0]};
+ true ->
+ update_docs_interactive(Db, Docs0, Options)
+ end.
+
+update_docs_interactive(Db, Docs0, Options) ->
Docs = tag_docs(Docs0),
AllOrNothing = lists:member(all_or_nothing, Options),
diff --git a/src/couch/src/couch_disk_monitor.erl b/src/couch/src/couch_disk_monitor.erl
new file mode 100644
index 000000000..22d5c83bd
--- /dev/null
+++ b/src/couch/src/couch_disk_monitor.erl
@@ -0,0 +1,241 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+ block_background_view_indexing/0,
+ block_interactive_view_indexing/0,
+ block_interactive_database_writes/0
+]).
+
+% testing
+-export([
+ set_database_dir_percent_used/1,
+ set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+ start_link/0,
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+ timer
+}).
+
+block_background_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled ->
+ view_index_dir_percent_used() > background_view_indexing_threshold();
+ true ->
+ false
+ end.
+
+block_interactive_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+ true -> false
+ end.
+
+block_interactive_database_writes() ->
+ Enabled = enabled(),
+ if
+ Enabled -> database_dir_percent_used() > interactive_database_writes_threshold();
+ true -> false
+ end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+ gen_server:call(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+ gen_server:call(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+ ets:new(?MODULE, [named_table, {read_concurrency, true}]),
+ ets:insert(?MODULE, {database_dir, 0}),
+ ets:insert(?MODULE, {view_index_dir, 0}),
+ update_disk_data(),
+ TRef = erlang:send_after(timer_interval(), self(), update_disk_data),
+ {ok, #st{timer = TRef}}.
+
+handle_call({set_database_dir_percent_used, PercentUsed}, _From, St) ->
+ ets:insert(?MODULE, {database_dir, PercentUsed}),
+ {reply, ok, St};
+handle_call({set_view_index_dir_percent_used, PercentUsed}, _From, St) ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+ {reply, ok, St};
+handle_call(_Msg, _From, St) ->
+ {reply, {error, unknown_msg}, St}.
+
+handle_cast(_Msg, St) ->
+ {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+ erlang:cancel_timer(St#st.timer),
+ update_disk_data(),
+ TRef = erlang:send_after(timer_interval(), self(), update_disk_data),
+ {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+ {noreply, St}.
+
+update_disk_data() ->
+ DiskData = disksup:get_disk_data(),
+ update_disk_data(DiskData).
+
+update_disk_data([]) ->
+ ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+ IsDatabaseDir = is_database_dir(Id),
+ IsViewIndexDir = is_view_index_dir(Id),
+ if
+ IsDatabaseDir ->
+ ets:insert(?MODULE, {database_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ if
+ IsViewIndexDir ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+ same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+ same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+ case {device_id(DirA), device_id(DirB)} of
+ {{ok, DeviceId}, {ok, DeviceId}} ->
+ true;
+ _Else ->
+ false
+ end.
+
+device_id(Dir) ->
+ case file:read_file_info(Dir) of
+ {ok, FileInfo} ->
+ {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+database_dir_percent_used() ->
+ [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+ PercentUsed.
+
+view_index_dir_percent_used() ->
+ [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+ PercentUsed.
+
+background_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "background_view_indexing_threshold", 80).
+
+interactive_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90).
+
+interactive_database_writes_threshold() ->
+ config:get_integer(?SECTION, "interactive_database_writes_threshold", 90).
+
+enabled() ->
+ config:get_boolean(?SECTION, "enable", false).
+
+%% Align our refresh with the os_mon refresh
+timer_interval() ->
+ disksup:get_check_interval().
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+not_enabled_by_default_test() ->
+ ?assertEqual(false, enabled()),
+ ?assertEqual(false, block_background_view_indexing()),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ ?assertEqual(false, block_interactive_database_writes()).
+
+setup_all() ->
+ Ctx = test_util:start_couch(),
+ config:set_boolean("disk_monitor", "enable", true, _Persist = false),
+ Ctx.
+
+teardown_all(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+setup() ->
+ ok.
+
+teardown(_) ->
+ ok.
+
+enabled_test_() ->
+ {
+ setup,
+ fun setup_all/0,
+ fun teardown_all/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(block_background_view_indexing_test),
+ ?TDEF_FE(block_interactive_view_indexing_test),
+ ?TDEF_FE(block_interactive_database_writes_test),
+ ?TDEF_FE(update_disk_data_test)
+ ]
+ }
+ }.
+
+block_background_view_indexing_test(_) ->
+ ?assert(enabled()),
+ set_view_index_dir_percent_used(80),
+ ?assertEqual(false, block_background_view_indexing()),
+ set_view_index_dir_percent_used(81),
+ ?assertEqual(true, block_background_view_indexing()).
+
+block_interactive_view_indexing_test(_) ->
+ ?assert(enabled()),
+ set_view_index_dir_percent_used(90),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ set_view_index_dir_percent_used(91),
+ ?assertEqual(true, block_interactive_view_indexing()).
+
+block_interactive_database_writes_test(_) ->
+ ?assert(enabled()),
+ set_database_dir_percent_used(90),
+ ?assertEqual(false, block_interactive_database_writes()),
+ set_database_dir_percent_used(91),
+ ?assertEqual(true, block_interactive_database_writes()).
+
+update_disk_data_test(_) ->
+ whereis(?MODULE) ! update_disk_data,
+ ?assertEqual({error, unknown_msg}, gen_server:call(?MODULE, foo)).
+
+-endif.
diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl
index cfe38bbd4..8fc3c9a13 100644
--- a/src/couch/src/couch_secondary_sup.erl
+++ b/src/couch/src/couch_secondary_sup.erl
@@ -26,7 +26,8 @@ init([]) ->
[
{query_servers, {couch_proc_manager, start_link, []}},
{vhosts, {couch_httpd_vhost, start_link, []}},
- {uuids, {couch_uuids, start, []}}
+ {uuids, {couch_uuids, start, []}},
+ {disk_manager, {couch_disk_monitor, start_link, []}}
] ++ couch_index_servers(),
MaybeHttp =
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 1a4a3ebcc..a50fcd670 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -291,8 +291,8 @@ query_view(Db, DDoc, VName, Args0, Callback, Acc0) ->
_ -> {ok, Acc0}
end,
query_view(Db, VInfo, Args, Callback, Acc1);
- ddoc_updated ->
- Callback(ok, ddoc_updated)
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Callback(ok, Error)
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -752,8 +752,8 @@ default_cb({final, Info}, []) ->
{ok, [Info]};
default_cb({final, _}, Acc) ->
{ok, Acc};
-default_cb(ok, ddoc_updated) ->
- {ok, ddoc_updated};
+default_cb(ok, Error) when Error == ddoc_updated; Error == insufficient_storage ->
+ {ok, Error};
default_cb(Row, Acc) ->
{ok, [Row | Acc]}.
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 5913aa3d0..a478685da 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -158,8 +158,8 @@ get_view(Db, DDoc, ViewName, Args0) ->
check_range(Args3, view_cmp(View)),
Sig = view_sig(Db, State, View, Args3),
{ok, {Type, View, Ref}, Sig, Args3};
- ddoc_updated ->
- ddoc_updated
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Error
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -176,6 +176,7 @@ get_view_index_state(_, DDoc, _, _, RetryCount) when RetryCount < 0 ->
couch_log:warning("DDoc '~s' recreated too frequently", [DDoc#doc.id]),
throw({get_view_state, exceeded_retry_count});
get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
+ BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(),
try
{ok, Pid, Args} = get_view_index_pid(Db, DDoc, ViewName, Args0),
UpdateSeq = couch_util:with_db(Db, fun(WDb) ->
@@ -190,12 +191,24 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
couch_index:get_state(Pid, 0);
false ->
couch_index:get_state(Pid, 0);
+ _ when BlockInteractiveIndexing ->
+ %% if the view is already fresh enough, proceed.
+ %% otherwise it's an error until there's room to update.
+ case couch_index:get_state(Pid, 0) of
+ {ok, #mrst{update_seq = MaybeSeq} = St} when MaybeSeq >= UpdateSeq ->
+ {ok, St};
+ {ok, #mrst{}} ->
+ insufficient_storage;
+ Else0 ->
+ Else0
+ end;
_ ->
couch_index:get_state(Pid, UpdateSeq)
end,
case State of
{ok, State0} -> {ok, State0, Args};
ddoc_updated -> ddoc_updated;
+ insufficient_storage -> insufficient_storage;
Else -> throw(Else)
end
catch
diff --git a/src/docs/src/config/disk-monitor.rst b/src/docs/src/config/disk-monitor.rst
new file mode 100644
index 000000000..ff6ad9e45
--- /dev/null
+++ b/src/docs/src/config/disk-monitor.rst
@@ -0,0 +1,76 @@
+.. Licensed under the Apache License, Version 2.0 (the "License"); you may not
+.. use this file except in compliance with the License. You may obtain a copy of
+.. the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+.. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+.. License for the specific language governing permissions and limitations under
+.. the License.
+
+.. default-domain:: config
+.. highlight:: ini
+
+==========================
+Disk Monitor Configuration
+==========================
+
+Apache CouchDB can react proactively when disk space gets low.
+
+.. _config/disk_monitor:
+
+Disk Monitor Options
+====================
+
+.. config:section:: disk_monitor :: Disk Monitor Options
+
+ .. versionadded:: 3.4
+
+ .. config:option:: background_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer start background view indexing jobs.
+ Defaults to ``80``. ::
+
+ [disk_monitor]
+ background_view_indexing_threshold = 80
+
+ .. config:option:: interactive_database_writes_threshold
+
+ The percentage of used disk space on the ``database_dir`` above
+ which CouchDB will no longer allow interactive document updates
+ (writes or deletes).
+
+ Replicated updates and database deletions are still permitted.
+
+ In a clustered write an error will be returned if
+ enough nodes are above the ``interactive_database_writes_threshold``.
+
+ Defaults to ``90``. ::
+
+ [disk_monitor]
+ interactive_database_writes_threshold = 90
+
+ .. config:option:: enable :: enable disk monitoring
+
+ Enable disk monitoring subsystem. Defaults to ``false``. ::
+
+ [disk_monitor]
+ enable = false
+
+ .. config:option:: interactive_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer update stale view indexes when queried.
+
+ View indexes that are already up to date can still be queried, and stale
+ view indexes can be queried if either ``stale=ok`` or ``update=false`` are
+ set.
+
+ Attempts to query a stale index without either parameter will yield a
+ ``507 Insufficient Storage`` error. Defaults to ``90``. ::
+
+ [disk_monitor]
+ interactive_view_indexing_threshold = 90
diff --git a/src/docs/src/config/index.rst b/src/docs/src/config/index.rst
index 7b5f1ba20..252697ffa 100644
--- a/src/docs/src/config/index.rst
+++ b/src/docs/src/config/index.rst
@@ -23,6 +23,7 @@ Configuration
couchdb
cluster
couch-peruser
+ disk-monitor
http
auth
compaction
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index b781eea99..5c1ddd573 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -493,7 +493,9 @@ view_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+view_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
reduce_cb({meta, Meta}, Acc) ->
% Map function starting
@@ -511,7 +513,9 @@ reduce_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
reduce_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+reduce_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
changes_enumerator(#full_doc_info{} = FDI, Acc) ->
changes_enumerator(couch_doc:to_doc_info(FDI), Acc);
diff --git a/src/fabric/src/fabric_streams.erl b/src/fabric/src/fabric_streams.erl
index 2a3a2b004..318824814 100644
--- a/src/fabric/src/fabric_streams.erl
+++ b/src/fabric/src/fabric_streams.erl
@@ -144,11 +144,11 @@ handle_stream_start(rexi_STREAM_INIT, {Worker, From}, St) ->
{stop, St#stream_acc{workers = [], ready = Ready1}}
end
end;
-handle_stream_start({ok, ddoc_updated}, _, St) ->
+handle_stream_start({ok, Error}, _, St) when Error == ddoc_updated; Error == insufficient_storage ->
WaitingWorkers = [W || {W, _} <- St#stream_acc.workers],
ReadyWorkers = [W || {W, _} <- St#stream_acc.ready],
cleanup(WaitingWorkers ++ ReadyWorkers),
- {stop, ddoc_updated};
+ {stop, Error};
handle_stream_start(Else, _, _) ->
exit({invalid_stream_start, Else}).
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 22535a9f9..87f464b36 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -419,7 +419,14 @@ maybe_update_others(
ViewName,
#mrargs{update = lazy} = Args
) ->
- ShardsNeedUpdated = mem3:shards(DbName) -- ShardsInvolved,
+ BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(),
+ ShardsNeedUpdated =
+ case BlockInteractiveIndexing of
+ true ->
+ [];
+ false ->
+ mem3:shards(DbName) -- ShardsInvolved
+ end,
lists:foreach(
fun(#shard{node = Node, name = ShardName}) ->
rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc, ViewName, Args])
diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl
index 7b3b75863..43922d5d5 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -56,6 +56,10 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback(
+ {error, {insufficient_storage, <<"not enough room to update index">>}}, Acc
+ );
{ok, Workers} ->
try
go(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -207,6 +211,8 @@ handle_message({execution_stats, _} = Msg, {_, From}, St) ->
rexi:stream_ack(From),
{Go, St#collector{user_acc = Acc}};
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
merge_row(Dir, Collation, undefined, Row, Rows0) ->
diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl
index 90fa523a1..d4d17d5e1 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -47,6 +47,8 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback({error, insufficient_storage}, Acc);
{ok, Workers} ->
try
go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -175,6 +177,8 @@ handle_message(complete, Worker, #collector{counters = Counters0} = State) ->
C1 = fabric_dict:update_counter(Worker, 1, Counters0),
fabric_view:maybe_send_row(State#collector{counters = C1});
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
os_proc_needed(<<"_", _/binary>>) -> false;
diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl
index 929ba47d7..c9cfdf31e 100644
--- a/src/ken/src/ken_server.erl
+++ b/src/ken/src/ken_server.erl
@@ -487,11 +487,16 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
IncrementalChannels = list_to_integer(config("incremental_channels", "80")),
BatchChannels = list_to_integer(config("batch_channels", "20")),
TotalChannels = IncrementalChannels + BatchChannels,
+ BlockBackgroundViewIndexing = couch_disk_monitor:block_background_view_indexing(),
+ % View name has two elements
+ IsView = tuple_size(Name) == 2,
A = get_active_count(),
#state{delay = Delay, batch_size = BS} = State,
case ets:lookup(ken_workers, Name) of
[] ->
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels ->
true;
A < TotalChannels ->
@@ -501,7 +506,7 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
{ok, CurrentSeq} = hastings_index:await(Pid, 0),
(Seq - CurrentSeq) < Threshold;
% View name has two elements.
- {_, _} ->
+ _ when IsView ->
% Since seq is 0, couch_index:get_state/2 won't
% spawn an index update.
{ok, MRSt} = couch_index:get_state(Pid, 0),
@@ -525,6 +530,8 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
Now = erlang:monotonic_time(),
DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond),
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels, (Seq - OldSeq) >= BS ->
true;
A < BatchChannels, DeltaT > Delay ->
diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir
index a79b3044b..1cbaae0f6 100644
--- a/test/elixir/test/config/suite.elixir
+++ b/test/elixir/test/config/suite.elixir
@@ -194,6 +194,11 @@
"design doc options - include_desing=true",
"design doc options - local_seq=true"
],
+ "DiskMonitorTest": [
+ "block background view indexing",
+ "block interactive view indexing",
+ "block interactive database writes"
+ ],
"DesignPathTest": [
"design doc path",
"design doc path with slash in db name"
diff --git a/test/elixir/test/disk_monitor.exs b/test/elixir/test/disk_monitor.exs
new file mode 100644
index 000000000..94a3edf99
--- /dev/null
+++ b/test/elixir/test/disk_monitor.exs
@@ -0,0 +1,55 @@
+defmodule DiskMonitorTest do
+ use CouchTestCase
+
+ @moduletag :disk_monitor
+
+ setup_all do
+ set_config({"disk_monitor", "enable", "true"})
+ :ok
+ end
+
+ @tag :with_db
+ test "block background view indexing", context do
+ set_config({"disk_monitor", "background_view_indexing_threshold", "0"})
+ set_config({"disk_monitor", "interactive_view_indexing_threshold", "100"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 201
+
+ map_doc = %{:views => %{:bar => %{:map => "function(doc) { emit(); }"}}}
+ assert Couch.put("/#{db_name}/_design/foo", body: map_doc).body["ok"]
+ :timer.sleep(500)
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar?stale=ok")
+ assert resp.body["total_rows"] == 0
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar")
+ assert resp.body["total_rows"] == 1
+ end
+
+ @tag :with_db
+ test "block interactive view indexing", context do
+ set_config({"disk_monitor", "background_view_indexing_threshold", "100"})
+ set_config({"disk_monitor", "interactive_view_indexing_threshold", "0"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 201
+
+ map_doc = %{:views => %{:bar => %{:map => "function(doc) { emit(); }"}}}
+ assert Couch.put("/#{db_name}/_design/foo", body: map_doc).body["ok"]
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar")
+ assert resp.status_code == 507
+ resp = Couch.get("/#{db_name}/_design/foo/_view/bar?stale=ok")
+ assert resp.status_code == 200
+ end
+
+ @tag :with_db
+ test "block interactive database writes", context do
+ set_config({"disk_monitor", "interactive_database_writes_threshold", "0"})
+
+ db_name = context[:db_name]
+ resp = Couch.post("/#{db_name}", body: %{:_id => "foo"})
+ assert resp.status_code == 507
+ end
+
+end