You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2023/07/19 17:18:18 UTC
[couchdb] branch out-of-disk-handler updated (7b86443b7 -> e2204b3bd)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch out-of-disk-handler
in repository https://gitbox.apache.org/repos/asf/couchdb.git
discard 7b86443b7 Introduce optional countermeasures as we run out of disk space
new e2204b3bd Introduce optional countermeasures as we run out of disk space
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7b86443b7)
\
N -- N -- N refs/heads/out-of-disk-handler (e2204b3bd)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/src/couch_disk_monitor.erl | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
[couchdb] 01/01: Introduce optional countermeasures as we run out of disk space
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch out-of-disk-handler
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e2204b3bd0747e7d478f7228a1ab5bc1d5a65be8
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Jun 16 09:25:41 2023 +0100
Introduce optional countermeasures as we run out of disk space
---
rel/overlay/etc/vm.args | 9 +
rel/reltool.config | 2 +
src/chttpd/src/chttpd.erl | 2 +
src/couch/priv/stats_descriptions.cfg | 4 +
src/couch/src/couch.app.src | 1 +
.../src/{couch_app.erl => couch_alarm_handler.erl} | 42 ++--
src/couch/src/couch_app.erl | 2 +
src/couch/src/couch_disk_monitor.erl | 258 +++++++++++++++++++++
src/couch/src/couch_secondary_sup.erl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 8 +-
src/couch_mrview/src/couch_mrview_util.erl | 17 +-
src/docs/src/config/disk-monitor.rst | 75 ++++++
src/docs/src/config/index.rst | 1 +
src/fabric/src/fabric_doc_update.erl | 19 +-
src/fabric/src/fabric_rpc.erl | 44 ++--
src/fabric/src/fabric_streams.erl | 4 +-
src/fabric/src/fabric_view.erl | 9 +-
src/fabric/src/fabric_view_map.erl | 6 +
src/fabric/src/fabric_view_reduce.erl | 4 +
src/ken/src/ken_server.erl | 8 +-
20 files changed, 461 insertions(+), 57 deletions(-)
diff --git a/rel/overlay/etc/vm.args b/rel/overlay/etc/vm.args
index 886bbb903..8b2440fc8 100644
--- a/rel/overlay/etc/vm.args
+++ b/rel/overlay/etc/vm.args
@@ -122,3 +122,12 @@
# in the features list.
#
#-crypto fips_mode true
+
+# OS Mon Settings
+
+# only start disksup
+-os_mon start_cpu_sup false
+-os_mon start_memsup false
+
+# Check disk space every 5 minutes
+-os_mon disk_space_check_interval 5
diff --git a/rel/reltool.config b/rel/reltool.config
index d84ef597c..ebb15bb83 100644
--- a/rel/reltool.config
+++ b/rel/reltool.config
@@ -23,6 +23,7 @@
sasl,
ssl,
stdlib,
+ os_mon,
syntax_tools,
xmerl,
%% couchdb
@@ -86,6 +87,7 @@
{app, sasl, [{incl_cond, include}]},
{app, ssl, [{incl_cond, include}]},
{app, stdlib, [{incl_cond, include}]},
+ {app, os_mon, [{incl_cond, include}]},
{app, syntax_tools, [{incl_cond, include}]},
{app, xmerl, [{incl_cond, include}]},
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 53abc731f..c8e6fdc97 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1138,6 +1138,8 @@ error_info(timeout) ->
>>};
error_info({service_unavailable, Reason}) ->
{503, <<"service unavailable">>, Reason};
+error_info({insufficient_storage, Reason}) ->
+ {507, <<"insufficent_storage">>, Reason};
error_info({timeout, _Reason}) ->
error_info(timeout);
error_info({'EXIT', {Error, _Stack}}) ->
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index 6c0d4dad2..1983eed9b 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -266,6 +266,10 @@
{type, counter},
{desc, <<"number of HTTP 503 Service unavailable responses">>}
]}.
+{[couchdb, httpd_status_codes, 507], [
+ {type, counter},
+ {desc, <<"number of HTTP 507 Insufficient Storage responses">>}
+]}.
{[couchdb, open_databases], [
{type, counter},
{desc, <<"number of open databases">>}
diff --git a/src/couch/src/couch.app.src b/src/couch/src/couch.app.src
index ef4e5e956..6fc293ac1 100644
--- a/src/couch/src/couch.app.src
+++ b/src/couch/src/couch.app.src
@@ -33,6 +33,7 @@
sasl,
inets,
ssl,
+ os_mon,
% Upstream deps
ibrowse,
diff --git a/src/couch/src/couch_app.erl b/src/couch/src/couch_alarm_handler.erl
similarity index 50%
copy from src/couch/src/couch_app.erl
copy to src/couch/src/couch_alarm_handler.erl
index 8cd8c8482..71f427128 100644
--- a/src/couch/src/couch_app.erl
+++ b/src/couch/src/couch_alarm_handler.erl
@@ -10,29 +10,31 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(couch_app).
-
--behaviour(application).
+-module(couch_alarm_handler).
+-behaviour(gen_event).
+% gen_event callbacks
-export([
- start/2,
- stop/1,
- uptime/0
+ init/1,
+ handle_event/2,
+ handle_call/2,
+ handle_info/2
]).
-start(_Type, _) ->
- case couch_sup:start_link() of
- {ok, _} = Resp ->
- {Time, _} = statistics(wall_clock),
- application:set_env(couch, start_time, Time),
- Resp;
- Else ->
- throw(Else)
- end.
+init(_Args) ->
+ {ok, nil}.
+
+handle_event({set_alarm, Alarm}, St) ->
+ error_logger:info_report([{alarm_handler, {set, Alarm}}]),
+ {ok, St};
+handle_event({clear_alarm, AlarmId}, St) ->
+ error_logger:info_report([{alarm_handler, {clear, AlarmId}}]),
+ {ok, St};
+handle_event(_Event, St) ->
+ {ok, St}.
-stop(_) ->
- ok.
+handle_call(_Query, St) ->
+ {ok, {error, bad_query}, St}.
-uptime() ->
- {Time, _} = statistics(wall_clock),
- Time - application:get_env(couch, start_time, Time).
+handle_info(_Msg, St) ->
+ {ok, St}.
diff --git a/src/couch/src/couch_app.erl b/src/couch/src/couch_app.erl
index 8cd8c8482..46476d634 100644
--- a/src/couch/src/couch_app.erl
+++ b/src/couch/src/couch_app.erl
@@ -21,6 +21,8 @@
]).
start(_Type, _) ->
+ %% register our alarm handler
+ gen_event:swap_handler(alarm_handler, {alarm_handler, swap}, {couch_alarm_handler, ok}),
case couch_sup:start_link() of
{ok, _} = Resp ->
{Time, _} = statistics(wall_clock),
diff --git a/src/couch/src/couch_disk_monitor.erl b/src/couch/src/couch_disk_monitor.erl
new file mode 100644
index 000000000..1eab80258
--- /dev/null
+++ b/src/couch/src/couch_disk_monitor.erl
@@ -0,0 +1,258 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_disk_monitor).
+-behaviour(gen_server).
+
+% public api
+-export([
+ block_background_view_indexing/0,
+ block_interactive_view_indexing/0,
+ block_database_writes/0
+]).
+
+% testing
+-export([
+ set_database_dir_percent_used/1,
+ set_view_index_dir_percent_used/1
+]).
+
+% gen_server callbacks
+-export([
+ start_link/0,
+ init/1,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2
+]).
+
+-include_lib("kernel/include/file.hrl").
+
+-define(SECTION, "disk_monitor").
+
+-record(st, {
+ timer
+}).
+
+block_background_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled ->
+ view_index_dir_percent_used() > background_view_indexing_threshold();
+ true ->
+ false
+ end.
+
+block_interactive_view_indexing() ->
+ Enabled = enabled(),
+ if
+ Enabled -> view_index_dir_percent_used() > interactive_view_indexing_threshold();
+ true -> false
+ end.
+
+block_database_writes() ->
+ Enabled = enabled(),
+ if
+ Enabled -> database_dir_percent_used() > database_writes_threshold();
+ true -> false
+ end.
+
+set_database_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+ gen_server:cast(?MODULE, {set_database_dir_percent_used, PercentUsed}).
+
+set_view_index_dir_percent_used(PercentUsed) when PercentUsed >= 0, PercentUsed =< 100 ->
+ gen_server:cast(?MODULE, {set_view_index_dir_percent_used, PercentUsed}).
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_Args) ->
+ ets:new(?MODULE, [named_table]),
+ update_disk_data(),
+ {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+ {ok, #st{timer = TRef}}.
+
+handle_call(_Msg, _From, St) ->
+ {reply, {error, unknown_msg}, St}.
+
+handle_cast({set_database_dir_percent_used, PercentUsed}, St) ->
+ ets:insert(?MODULE, {database_dir, PercentUsed}),
+ {noreply, St};
+handle_cast({set_view_index_dir_percent_used, PercentUsed}, St) ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed}),
+ {noreply, St};
+handle_cast(_Msg, St) ->
+ {noreply, St}.
+
+handle_info(update_disk_data, St) ->
+ timer:cancel(St#st.timer),
+ update_disk_data(),
+ {ok, TRef} = timer:send_after(timer_interval(), update_disk_data),
+ {noreply, St#st{timer = TRef}};
+handle_info(_Msg, St) ->
+ {noreply, St}.
+
+update_disk_data() ->
+ DiskData = disksup:get_disk_data(),
+ update_disk_data(DiskData).
+
+update_disk_data([]) ->
+ ok;
+update_disk_data([{Id, _, PercentUsed} | Rest]) ->
+ IsDatabaseDir = is_database_dir(Id),
+ IsViewIndexDir = is_view_index_dir(Id),
+ if
+ IsDatabaseDir ->
+ ets:insert(?MODULE, {database_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ if
+ IsViewIndexDir ->
+ ets:insert(?MODULE, {view_index_dir, PercentUsed});
+ true ->
+ ok
+ end,
+ update_disk_data(Rest).
+
+is_database_dir(MntOn) ->
+ same_device(config:get("couchdb", "database_dir"), MntOn).
+
+is_view_index_dir(MntOn) ->
+ same_device(config:get("couchdb", "view_index_dir"), MntOn).
+
+same_device(DirA, DirB) ->
+ case {device_id(DirA), device_id(DirB)} of
+ {{ok, DeviceId}, {ok, DeviceId}} ->
+ true;
+ _Else ->
+ false
+ end.
+
+device_id(Dir) ->
+ case file:read_file_info(Dir) of
+ {ok, FileInfo} ->
+ {ok, {FileInfo#file_info.minor_device, FileInfo#file_info.major_device}};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+database_dir_percent_used() ->
+ [{database_dir, PercentUsed}] = ets:lookup(?MODULE, database_dir),
+ PercentUsed.
+
+view_index_dir_percent_used() ->
+ [{view_index_dir, PercentUsed}] = ets:lookup(?MODULE, view_index_dir),
+ PercentUsed.
+
+background_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "background_view_indexing_threshold", 80).
+
+interactive_view_indexing_threshold() ->
+ config:get_integer(?SECTION, "interactive_view_indexing_threshold", 90).
+
+database_writes_threshold() ->
+ config:get_integer(?SECTION, "database_writes_threshold", 90).
+
+enabled() ->
+ config:get_boolean(?SECTION, "enable", false).
+
+%% Align our refresh with the os_mon refresh
+timer_interval() ->
+ application:get_env(os_mon, disk_space_check_interval, 30) * 60_000.
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+not_enabled_by_default_test() ->
+ ?assertEqual(false, enabled()),
+ ?assertEqual(false, block_background_view_indexing()),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ ?assertEqual(false, block_database_writes()).
+
+setup_all() ->
+ application:ensure_all_started(os_mon),
+ {ok, CurrentDirectory} = file:get_cwd(),
+ ok = meck:new(config, [passthrough]),
+ ok = meck:expect(
+ config,
+ get,
+ fun
+ ("couchdb", "database_dir") -> CurrentDirectory;
+ ("couchdb", "view_index_dir") -> CurrentDirectory
+ end
+ ),
+ ok = meck:expect(config, get_boolean, ["disk_monitor", "enable", '_'], true),
+ ok = meck:expect(
+ config,
+ get_integer,
+ fun
+ ("disk_monitor", "background_view_indexing_threshold", _) -> 50;
+ ("disk_monitor", "interactive_view_indexing_threshold", _) -> 60;
+ ("disk_monitor", "database_writes_threshold", _) -> 70
+ end
+ ),
+ {ok, Pid} = start_link(),
+ Pid.
+
+teardown_all(Pid) ->
+ true = exit(Pid, normal),
+ application:stop(os_mon),
+ meck:unload().
+
+setup() ->
+ meck:reset([config]).
+
+teardown(_) ->
+ ok.
+
+enabled_test_() ->
+ {
+ setup,
+ fun setup_all/0,
+ fun teardown_all/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(block_background_view_indexing_test),
+ ?TDEF_FE(block_interactive_view_indexing_test),
+ ?TDEF_FE(block_database_writes_test),
+ ?TDEF_FE(update_disk_data_test)
+ ]
+ }
+ }.
+
+block_background_view_indexing_test(_) ->
+ set_view_index_dir_percent_used(50),
+ ?assertEqual(false, block_background_view_indexing()),
+ set_view_index_dir_percent_used(51),
+ ?assertEqual(true, block_background_view_indexing()).
+
+block_interactive_view_indexing_test(_) ->
+ set_view_index_dir_percent_used(60),
+ ?assertEqual(false, block_interactive_view_indexing()),
+ set_view_index_dir_percent_used(61),
+ ?assertEqual(true, block_interactive_view_indexing()).
+
+block_database_writes_test(_) ->
+ set_database_dir_percent_used(70),
+ ?assertEqual(false, block_database_writes()),
+ set_database_dir_percent_used(71),
+ ?assertEqual(true, block_database_writes()).
+
+update_disk_data_test(_) ->
+ whereis(?MODULE) ! update_disk_data,
+ ?assertEqual({error, unknown_msg}, gen_server:call(?MODULE, foo)).
+
+-endif.
diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl
index cfe38bbd4..8fc3c9a13 100644
--- a/src/couch/src/couch_secondary_sup.erl
+++ b/src/couch/src/couch_secondary_sup.erl
@@ -26,7 +26,8 @@ init([]) ->
[
{query_servers, {couch_proc_manager, start_link, []}},
{vhosts, {couch_httpd_vhost, start_link, []}},
- {uuids, {couch_uuids, start, []}}
+ {uuids, {couch_uuids, start, []}},
+ {disk_manager, {couch_disk_monitor, start_link, []}}
] ++ couch_index_servers(),
MaybeHttp =
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 1a4a3ebcc..a50fcd670 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -291,8 +291,8 @@ query_view(Db, DDoc, VName, Args0, Callback, Acc0) ->
_ -> {ok, Acc0}
end,
query_view(Db, VInfo, Args, Callback, Acc1);
- ddoc_updated ->
- Callback(ok, ddoc_updated)
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Callback(ok, Error)
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -752,8 +752,8 @@ default_cb({final, Info}, []) ->
{ok, [Info]};
default_cb({final, _}, Acc) ->
{ok, Acc};
-default_cb(ok, ddoc_updated) ->
- {ok, ddoc_updated};
+default_cb(ok, Error) when Error == ddoc_updated; Error == insufficient_storage ->
+ {ok, Error};
default_cb(Row, Acc) ->
{ok, [Row | Acc]}.
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 5913aa3d0..a478685da 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -158,8 +158,8 @@ get_view(Db, DDoc, ViewName, Args0) ->
check_range(Args3, view_cmp(View)),
Sig = view_sig(Db, State, View, Args3),
{ok, {Type, View, Ref}, Sig, Args3};
- ddoc_updated ->
- ddoc_updated
+ Error when Error == ddoc_updated; Error == insufficient_storage ->
+ Error
end.
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
@@ -176,6 +176,7 @@ get_view_index_state(_, DDoc, _, _, RetryCount) when RetryCount < 0 ->
couch_log:warning("DDoc '~s' recreated too frequently", [DDoc#doc.id]),
throw({get_view_state, exceeded_retry_count});
get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
+ BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(),
try
{ok, Pid, Args} = get_view_index_pid(Db, DDoc, ViewName, Args0),
UpdateSeq = couch_util:with_db(Db, fun(WDb) ->
@@ -190,12 +191,24 @@ get_view_index_state(Db, DDoc, ViewName, Args0, RetryCount) ->
couch_index:get_state(Pid, 0);
false ->
couch_index:get_state(Pid, 0);
+ _ when BlockInteractiveIndexing ->
+ %% if the view is already fresh enough, proceed.
+ %% otherwise it's an error until there's room to update.
+ case couch_index:get_state(Pid, 0) of
+ {ok, #mrst{update_seq = MaybeSeq} = St} when MaybeSeq >= UpdateSeq ->
+ {ok, St};
+ {ok, #mrst{}} ->
+ insufficient_storage;
+ Else0 ->
+ Else0
+ end;
_ ->
couch_index:get_state(Pid, UpdateSeq)
end,
case State of
{ok, State0} -> {ok, State0, Args};
ddoc_updated -> ddoc_updated;
+ insufficient_storage -> insufficient_storage;
Else -> throw(Else)
end
catch
diff --git a/src/docs/src/config/disk-monitor.rst b/src/docs/src/config/disk-monitor.rst
new file mode 100644
index 000000000..70af6be0c
--- /dev/null
+++ b/src/docs/src/config/disk-monitor.rst
@@ -0,0 +1,75 @@
+.. Licensed under the Apache License, Version 2.0 (the "License"); you may not
+.. use this file except in compliance with the License. You may obtain a copy of
+.. the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+.. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+.. License for the specific language governing permissions and limitations under
+.. the License.
+
+.. default-domain:: config
+.. highlight:: ini
+
+==========================
+Disk Monitor Configuration
+==========================
+
+Apache CouchDB can react proactively when disk space gets low.
+
+.. _config/disk_monitor:
+
+Disk Monitor Options
+====================
+
+.. config:section:: disk_monitor :: Disk Monitor Options
+
+ .. versionadded:: 3.4
+
+ .. config:option:: background_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer start background view indexing jobs.
+ Defaults to ``80``. ::
+
+ [disk_monitor]
+ background_view_indexing_threshold = 80
+
+ .. config:option:: database_writes_threshold
+
+ The percentage of used disk space on the ``database_dir`` above
+ which CouchDB will no longer allow document updates (writes or deletes).
+
+ Database deletions are still permitted.
+
+ In a clustered write a ``500 Internal Server`` error will be returned if
+ enough nodes are above the ``database_writes_threshold``.
+
+ Defaults to ``90``. ::
+
+ [disk_monitor]
+ database_writes_threshold = 90
+
+ .. config:option:: enable :: enable disk monitoring
+
+ Enable disk monitoring subsystem. Defaults to ``false``. ::
+
+ [disk_monitor]
+ enable = false
+
+ .. config:option:: interactive_view_indexing_threshold
+
+ The percentage of used disk space on the ``view_index_dir`` above
+ which CouchDB will no longer update stale view indexes when queried.
+
+ View indexes that are already up to date can still be queried, and stale
+ view indexes can be queried if either ``stale=ok`` or ``update=false`` are
+ set.
+
+ Attempts to query a stale index without either parameter will yield a
+ ``507 Insufficient Storage`` error. Defaults to ``90``. ::
+
+ [disk_monitor]
+ interactive_view_indexing_threshold = 90
diff --git a/src/docs/src/config/index.rst b/src/docs/src/config/index.rst
index 7b5f1ba20..252697ffa 100644
--- a/src/docs/src/config/index.rst
+++ b/src/docs/src/config/index.rst
@@ -23,6 +23,7 @@ Configuration
couchdb
cluster
couch-peruser
+ disk-monitor
http
auth
compaction
diff --git a/src/fabric/src/fabric_doc_update.erl b/src/fabric/src/fabric_doc_update.erl
index 77b424911..44448c742 100644
--- a/src/fabric/src/fabric_doc_update.erl
+++ b/src/fabric/src/fabric_doc_update.erl
@@ -63,19 +63,15 @@ handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, Acc0) ->
NewGrpDocs = [X || {#shard{node = N}, _} = X <- GroupedDocs, N =/= NodeRef],
skip_message({length(NewGrpDocs), LenDocs, W, NewGrpDocs, DocReplyDict});
handle_message({rexi_EXIT, _}, Worker, Acc0) ->
- {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0,
- NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs),
- skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict});
+ ignore_failed_worker(Worker, Acc0);
handle_message({error, all_dbs_active}, Worker, Acc0) ->
% treat it like rexi_EXIT, the hope at least one copy will return successfully
- {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0,
- NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs),
- skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict});
+ ignore_failed_worker(Worker, Acc0);
handle_message(internal_server_error, Worker, Acc0) ->
% happens when we fail to load validation functions in an RPC worker
- {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0,
- NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs),
- skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict});
+ ignore_failed_worker(Worker, Acc0);
+handle_message({insufficient_storage, _Msg}, Worker, Acc0) ->
+ ignore_failed_worker(Worker, Acc0);
handle_message(attachment_chunk_received, _Worker, Acc0) ->
{ok, Acc0};
handle_message({ok, Replies}, Worker, Acc0) ->
@@ -115,6 +111,11 @@ handle_message({forbidden, Msg}, _, _) ->
handle_message({request_entity_too_large, Entity}, _, _) ->
throw({request_entity_too_large, Entity}).
+ignore_failed_worker(Worker, Acc0) ->
+ {WC, LenDocs, W, GrpDocs, DocReplyDict} = Acc0,
+ NewGrpDocs = lists:keydelete(Worker, 1, GrpDocs),
+ skip_message({WC - 1, LenDocs, W, NewGrpDocs, DocReplyDict}).
+
before_doc_update(DbName, Docs, Opts) ->
% Use the same pattern as in couch_db:validate_doc_update/3. If the document was already
% checked during the interactive edit we don't want to spend time in the internal replicator
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index b781eea99..fa75370a3 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -274,21 +274,27 @@ get_missing_revs(DbName, IdRevsList, Options) ->
with_db(DbName, Options, {couch_db, get_missing_revs, [IdRevsList]}).
update_docs(DbName, Docs0, Options) ->
- {Docs1, Type} =
- case couch_util:get_value(read_repair, Options) of
- NodeRevs when is_list(NodeRevs) ->
- Filtered = read_repair_filter(DbName, Docs0, NodeRevs, Options),
- {Filtered, ?REPLICATED_CHANGES};
- undefined ->
- X =
- case proplists:get_value(?REPLICATED_CHANGES, Options) of
- true -> ?REPLICATED_CHANGES;
- _ -> ?INTERACTIVE_EDIT
- end,
- {Docs0, X}
- end,
- Docs2 = make_att_readers(Docs1),
- with_db(DbName, Options, {couch_db, update_docs, [Docs2, Options, Type]}).
+ %% only if there's room
+ case couch_disk_monitor:block_database_writes() of
+ true ->
+ rexi:reply({insufficient_storage, <<"database_dir almost full">>});
+ false ->
+ {Docs1, Type} =
+ case couch_util:get_value(read_repair, Options) of
+ NodeRevs when is_list(NodeRevs) ->
+ Filtered = read_repair_filter(DbName, Docs0, NodeRevs, Options),
+ {Filtered, ?REPLICATED_CHANGES};
+ undefined ->
+ X =
+ case proplists:get_value(?REPLICATED_CHANGES, Options) of
+ true -> ?REPLICATED_CHANGES;
+ _ -> ?INTERACTIVE_EDIT
+ end,
+ {Docs0, X}
+ end,
+ Docs2 = make_att_readers(Docs1),
+ with_db(DbName, Options, {couch_db, update_docs, [Docs2, Options, Type]})
+ end.
get_purge_seq(DbName, Options) ->
with_db(DbName, Options, {couch_db, get_purge_seq, []}).
@@ -493,7 +499,9 @@ view_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+view_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
reduce_cb({meta, Meta}, Acc) ->
% Map function starting
@@ -511,7 +519,9 @@ reduce_cb(complete, Acc) ->
ok = rexi:stream_last(complete),
{ok, Acc};
reduce_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
+ rexi:reply({ok, ddoc_updated});
+reduce_cb(ok, insufficient_storage) ->
+ rexi:reply({ok, insufficient_storage}).
changes_enumerator(#full_doc_info{} = FDI, Acc) ->
changes_enumerator(couch_doc:to_doc_info(FDI), Acc);
diff --git a/src/fabric/src/fabric_streams.erl b/src/fabric/src/fabric_streams.erl
index 2a3a2b004..318824814 100644
--- a/src/fabric/src/fabric_streams.erl
+++ b/src/fabric/src/fabric_streams.erl
@@ -144,11 +144,11 @@ handle_stream_start(rexi_STREAM_INIT, {Worker, From}, St) ->
{stop, St#stream_acc{workers = [], ready = Ready1}}
end
end;
-handle_stream_start({ok, ddoc_updated}, _, St) ->
+handle_stream_start({ok, Error}, _, St) when Error == ddoc_updated; Error == insufficient_storage ->
WaitingWorkers = [W || {W, _} <- St#stream_acc.workers],
ReadyWorkers = [W || {W, _} <- St#stream_acc.ready],
cleanup(WaitingWorkers ++ ReadyWorkers),
- {stop, ddoc_updated};
+ {stop, Error};
handle_stream_start(Else, _, _) ->
exit({invalid_stream_start, Else}).
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 82f9c2bc6..bad5e32c2 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -415,7 +415,14 @@ maybe_update_others(
ViewName,
#mrargs{update = lazy} = Args
) ->
- ShardsNeedUpdated = mem3:shards(DbName) -- ShardsInvolved,
+ BlockInteractiveIndexing = couch_disk_monitor:block_interactive_view_indexing(),
+ ShardsNeedUpdated =
+ case BlockInteractiveIndexing of
+ true ->
+ [];
+ false ->
+ mem3:shards(DbName) -- ShardsInvolved
+ end,
lists:foreach(
fun(#shard{node = Node, name = ShardName}) ->
rpc:cast(Node, fabric_rpc, update_mrview, [ShardName, DDoc, ViewName, Args])
diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl
index 7b3b75863..43922d5d5 100644
--- a/src/fabric/src/fabric_view_map.erl
+++ b/src/fabric/src/fabric_view_map.erl
@@ -56,6 +56,10 @@ go(Db, Options, DDoc, View, Args0, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback(
+ {error, {insufficient_storage, <<"not enough room to update index">>}}, Acc
+ );
{ok, Workers} ->
try
go(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -207,6 +211,8 @@ handle_message({execution_stats, _} = Msg, {_, From}, St) ->
rexi:stream_ack(From),
{Go, St#collector{user_acc = Acc}};
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
merge_row(Dir, Collation, undefined, Row, Rows0) ->
diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl
index 90fa523a1..d4d17d5e1 100644
--- a/src/fabric/src/fabric_view_reduce.erl
+++ b/src/fabric/src/fabric_view_reduce.erl
@@ -47,6 +47,8 @@ go(Db, DDoc, VName, Args, Callback, Acc, VInfo) ->
of
{ok, ddoc_updated} ->
Callback({error, ddoc_updated}, Acc);
+ {ok, insufficient_storage} ->
+ Callback({error, insufficient_storage}, Acc);
{ok, Workers} ->
try
go2(DbName, Workers, VInfo, CoordArgs, Callback, Acc)
@@ -175,6 +177,8 @@ handle_message(complete, Worker, #collector{counters = Counters0} = State) ->
C1 = fabric_dict:update_counter(Worker, 1, Counters0),
fabric_view:maybe_send_row(State#collector{counters = C1});
handle_message(ddoc_updated, _Worker, State) ->
+ {stop, State};
+handle_message(insufficient_storage, _Worker, State) ->
{stop, State}.
os_proc_needed(<<"_", _/binary>>) -> false;
diff --git a/src/ken/src/ken_server.erl b/src/ken/src/ken_server.erl
index 929ba47d7..f1d838a74 100644
--- a/src/ken/src/ken_server.erl
+++ b/src/ken/src/ken_server.erl
@@ -487,11 +487,15 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
IncrementalChannels = list_to_integer(config("incremental_channels", "80")),
BatchChannels = list_to_integer(config("batch_channels", "20")),
TotalChannels = IncrementalChannels + BatchChannels,
+ BlockBackgroundViewIndexing = couch_disk_monitor:block_background_view_indexing(),
+ IsView = tuple_size(Name) == 2, % View name has two elements
A = get_active_count(),
#state{delay = Delay, batch_size = BS} = State,
case ets:lookup(ken_workers, Name) of
[] ->
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels ->
true;
A < TotalChannels ->
@@ -501,7 +505,7 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
{ok, CurrentSeq} = hastings_index:await(Pid, 0),
(Seq - CurrentSeq) < Threshold;
% View name has two elements.
- {_, _} ->
+ _ when IsView ->
% Since seq is 0, couch_index:get_state/2 won't
% spawn an index update.
{ok, MRSt} = couch_index:get_state(Pid, 0),
@@ -525,6 +529,8 @@ should_start_job(#job{name = Name, seq = Seq, server = Pid}, State) ->
Now = erlang:monotonic_time(),
DeltaT = erlang:convert_time_unit(Now - LRU, native, millisecond),
if
+ BlockBackgroundViewIndexing andalso IsView ->
+ false;
A < BatchChannels, (Seq - OldSeq) >= BS ->
true;
A < BatchChannels, DeltaT > Delay ->