You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2020/01/06 18:27:00 UTC
[couchdb] branch ioq-in-tree created (now 3c50283)
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a change to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 3c50283 Remove ioq as dependency and add as subdir
This branch includes the following new commits:
new 6293a8b Initial commit
new 04aea73 new IOQ api
new c7f9ad1 Fallback to direct I/O if ioq is not running
new 40d157f Update state on config changes
new c552c66 Allow to customize concurrency value
new 2175035 Handle {gen_event_EXIT,{config_listener,ioq},shutdown} message
new ad60d32 Update config_listener behaviuor
new 0ffa7cd Don't restart event handler on termination
new c7c75eb Merge remote-tracking branch 'iilyak/2561-make-config-API-consistent'
new bdcfe6a Update handle_config_terminate API
new 126a849 Fix a typo in a child name
new ba99ec7 Merge remote branch 'cloudant:3102-fix-config_subscription'
new 5f5375a Remove unused code
new 1d2b149 Merge remote branch 'cloudant:remove-unused-config-subscriber'
new 345804c Use couch_rand compatibility module
new 04bebb3 Merge branch 'use-couch-rand-module'
new e641a74 Enable users to bypass IOQ for certain IO classes
new 5cccb9d Add 'src/ioq/' from commit 'e641a740978447f0b29785580e46d2e30e822001'
new 3c50283 Remove ioq as dependency and add as subdir
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 14/19: Merge remote branch
'cloudant:remove-unused-config-subscriber'
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1d2b149ee12dfeaf8d89a67b2f937207f4c5bdf2
Merge: ba99ec7 5f5375a
Author: Eric Avdey <ei...@eiri.ca>
AuthorDate: Tue Oct 4 13:10:57 2016 -0300
Merge remote branch 'cloudant:remove-unused-config-subscriber'
This closes #5
Signed-off-by: Eric Avdey <ei...@eiri.ca>
src/ioq_sup.erl | 23 +----------------------
1 file changed, 1 insertion(+), 22 deletions(-)
[couchdb] 07/19: Update config_listener behaviuor
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ad60d329a038d2b1aa5ee083f22b1ee7906ec31d
Author: ILYA Khlopotov <ii...@ca.ibm.com>
AuthorDate: Fri Jan 30 10:59:43 2015 -0800
Update config_listener behaviuor
COUCHDB-2561
---
src/ioq.erl | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 6c01b9c..c4b3b4e 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -18,7 +18,7 @@
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]).
% config_listener api
--export([handle_config_change/5]).
+-export([handle_config_change/5, handle_config_terminate/3]).
-record(state, {
concurrency,
@@ -83,12 +83,6 @@ handle_info({'DOWN', Ref, _, _, Reason}, State) ->
false ->
{noreply, State, 0}
end;
-handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
- erlang:send_after(5000, self(), restart_config_listener),
- {noreply, State};
-handle_info(restart_config_listener, State) ->
- ok = config:listen_for_changes(?MODULE, nil),
- {noreply, State};
handle_info(timeout, State) ->
{noreply, maybe_submit_request(State)}.
@@ -97,6 +91,13 @@ handle_config_change("ioq", _, _, _, _) ->
handle_config_change(_, _, _, _, _) ->
{ok, nil}.
+handle_config_terminate(_, _, _) ->
+ spawn(fun() ->
+ timer:sleep(5000),
+ config:listen_for_changes(?MODULE, nil)
+ end),
+ ok.
+
code_change(_Vsn, State, _Extra) ->
{ok, State}.
[couchdb] 11/19: Fix a typo in a child name
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 126a849fa394f336bc769e85adad143a651c4ec1
Author: ILYA Khlopotov <ii...@ca.ibm.com>
AuthorDate: Tue Aug 23 14:23:24 2016 -0700
Fix a typo in a child name
COUCHDB-3102
---
src/ioq_sup.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/ioq_sup.erl b/src/ioq_sup.erl
index 56e51ae..fa919e4 100644
--- a/src/ioq_sup.erl
+++ b/src/ioq_sup.erl
@@ -31,7 +31,7 @@ init([]) ->
worker,
[config_listener_mon]
},
- ?CHILD(ioq_server, worker)
+ ?CHILD(ioq, worker)
]} }.
handle_config_change("ioq", _Key, _Val, _Persist, St) ->
[couchdb] 03/19: Fallback to direct I/O if ioq is not running
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c7f9ad1c0f77b601aa456e62c963793d517c1026
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Sep 5 18:22:44 2014 +0100
Fallback to direct I/O if ioq is not running
---
src/ioq.erl | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 4983b73..b761a0b 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -37,7 +37,12 @@ start_link() ->
call(Fd, Msg, Priority) ->
Request = #request{fd=Fd, msg=Msg, priority=Priority, from=self()},
- gen_server:call(?MODULE, Request, infinity).
+ try
+ gen_server:call(?MODULE, Request, infinity)
+ catch
+ exit:{noproc,_} ->
+ gen_server:call(Fd, Msg, infinity)
+ end.
init(_) ->
Ratio = list_to_float(config:get("ioq", "ratio", "0.01")),
[couchdb] 18/19: Add 'src/ioq/' from commit
'e641a740978447f0b29785580e46d2e30e822001'
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5cccb9dbf0be7bfb9133cf157cf4572ccde601ae
Merge: 05b5d06 e641a74
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Mon Jan 6 13:24:35 2020 -0500
Add 'src/ioq/' from commit 'e641a740978447f0b29785580e46d2e30e822001'
git-subtree-dir: src/ioq
git-subtree-mainline: 05b5d063c93c1c7c73bbcb6af601b1a7f796233d
git-subtree-split: e641a740978447f0b29785580e46d2e30e822001
src/ioq/.gitignore | 2 +
src/ioq/src/ioq.app.src | 21 ++++++
src/ioq/src/ioq.erl | 189 ++++++++++++++++++++++++++++++++++++++++++++++++
src/ioq/src/ioq_app.erl | 21 ++++++
src/ioq/src/ioq_sup.erl | 24 ++++++
5 files changed, 257 insertions(+)
diff --cc src/ioq/.gitignore
index 0000000,21cf3d3..21cf3d3
mode 000000,100644..100644
--- a/src/ioq/.gitignore
+++ b/src/ioq/.gitignore
diff --cc src/ioq/src/ioq.app.src
index 0000000,65ea50d..65ea50d
mode 000000,100644..100644
--- a/src/ioq/src/ioq.app.src
+++ b/src/ioq/src/ioq.app.src
diff --cc src/ioq/src/ioq.erl
index 0000000,81d94a3..81d94a3
mode 000000,100644..100644
--- a/src/ioq/src/ioq.erl
+++ b/src/ioq/src/ioq.erl
diff --cc src/ioq/src/ioq_app.erl
index 0000000,2e6d75a..2e6d75a
mode 000000,100644..100644
--- a/src/ioq/src/ioq_app.erl
+++ b/src/ioq/src/ioq_app.erl
diff --cc src/ioq/src/ioq_sup.erl
index 0000000,c4d04a9..c4d04a9
mode 000000,100644..100644
--- a/src/ioq/src/ioq_sup.erl
+++ b/src/ioq/src/ioq_sup.erl
[couchdb] 17/19: Enable users to bypass IOQ for certain IO classes
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e641a740978447f0b29785580e46d2e30e822001
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Wed Dec 18 15:22:18 2019 -0500
Enable users to bypass IOQ for certain IO classes
This patch allows an administrator to configure a "bypass" which
will cause a particular class of IO to be submitted directly to the
file descriptor or OS process instead of going through the IO queueing
mechanism. Installing a bypass can result in higher throughput and
lower latency, at the expense of less control over the stability of the
system.
A bypass is configured via the `ioq.priority` configuration block:
[ioq.bypass]
read = true
write = true
compaction = false
This configuration will cause user-submitted read IO to be submitted
directly. At this time the following classes are available:
- os_process
- read
- write
- view_update
- shard_sync
- compaction
This also expands the "compaction" queue to be a general-purpose
"background" queue that handles IO for both compaction and internal
replication (aka shard_sync). The other four classes are handled by the
"interactive" queue. As before, the [ioq] ratio setting determines the
likelihood that background IO will be selected ahead of interactive IO
when both queues are non-empty.
---
src/ioq.erl | 49 ++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 40 insertions(+), 9 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 9ca2656..81d94a3 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -26,7 +26,7 @@
concurrency,
ratio,
interactive=queue:new(),
- compaction=queue:new(),
+ background=queue:new(),
running=[]
}).
@@ -41,7 +41,38 @@
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-call(Fd, Msg, Priority) ->
+call(Fd, Msg, Metadata) ->
+ Priority = io_class(Msg, Metadata),
+ case bypass(Priority) of
+ true ->
+ gen_server:call(Fd, Msg);
+ false ->
+ queued_call(Fd, Msg, Priority)
+ end.
+
+bypass(Priority) ->
+ config:get("ioq.bypass", atom_to_list(Priority)) =:= "true".
+
+io_class({prompt, _}, _) ->
+ os_process;
+io_class({data, _}, _) ->
+ os_process;
+io_class(_, {interactive, _}) ->
+ read;
+io_class(_, {db_update, _}) ->
+ write;
+io_class(_, {view_update, _, _}) ->
+ view_update;
+io_class(_, {internal_repl, _}) ->
+ shard_sync;
+io_class(_, {db_compact, _}) ->
+ compaction;
+io_class(_, {view_compact, _, _}) ->
+ compaction;
+io_class(_, _) ->
+ other.
+
+queued_call(Fd, Msg, Priority) ->
Request = #request{fd=Fd, msg=Msg, priority=Priority, from=self()},
try
gen_server:call(?MODULE, Request, infinity)
@@ -107,10 +138,10 @@ code_change(_Vsn, State, _Extra) ->
terminate(_Reason, _State) ->
ok.
-enqueue_request(#request{priority={db_compact, _}}=Request, #state{}=State) ->
- State#state{compaction=queue:in(Request, State#state.compaction)};
-enqueue_request(#request{priority={view_compact, _, _}}=Request, #state{}=State) ->
- State#state{compaction=queue:in(Request, State#state.compaction)};
+enqueue_request(#request{priority=compaction}=Request, #state{}=State) ->
+ State#state{background=queue:in(Request, State#state.background)};
+enqueue_request(#request{priority=shard_sync}=Request, #state{}=State) ->
+ State#state{background=queue:in(Request, State#state.background)};
enqueue_request(#request{}=Request, #state{}=State) ->
State#state{interactive=queue:in(Request, State#state.interactive)}.
@@ -128,17 +159,17 @@ maybe_submit_request(State) ->
State.
make_next_request(#state{}=State) ->
- case {queue:is_empty(State#state.compaction), queue:is_empty(State#state.interactive)} of
+ case {queue:is_empty(State#state.background), queue:is_empty(State#state.interactive)} of
{true, true} ->
State;
{true, false} ->
choose_next_request(#state.interactive, State);
{false, true} ->
- choose_next_request(#state.compaction, State);
+ choose_next_request(#state.background, State);
{false, false} ->
case couch_rand:uniform() < State#state.ratio of
true ->
- choose_next_request(#state.compaction, State);
+ choose_next_request(#state.background, State);
false ->
choose_next_request(#state.interactive, State)
end
[couchdb] 13/19: Remove unused code
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5f5375a0d4fb1ceae9a14ba28ad991ff020a9c9e
Author: Eric Avdey <ei...@eiri.ca>
AuthorDate: Mon Oct 3 11:42:02 2016 -0300
Remove unused code
We are subscribing both ioq and ioq_sup to config_event,
but while ioq is actually processing config changes,
ioq_sup just sends uncatched messages to unexisting ioq_server.
---
src/ioq_sup.erl | 23 +----------------------
1 file changed, 1 insertion(+), 22 deletions(-)
diff --git a/src/ioq_sup.erl b/src/ioq_sup.erl
index fa919e4..c4d04a9 100644
--- a/src/ioq_sup.erl
+++ b/src/ioq_sup.erl
@@ -13,7 +13,6 @@
-module(ioq_sup).
-behaviour(supervisor).
-export([start_link/0, init/1]).
--export([handle_config_change/5, handle_config_terminate/3]).
%% Helper macro for declaring children of supervisor
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
@@ -22,24 +21,4 @@ start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->
- {ok, { {one_for_one, 5, 10}, [
- {
- config_listener_mon,
- {config_listener_mon, start_link, [?MODULE, nil]},
- permanent,
- 5000,
- worker,
- [config_listener_mon]
- },
- ?CHILD(ioq, worker)
- ]} }.
-
-handle_config_change("ioq", _Key, _Val, _Persist, St) ->
- gen_server:cast(ioq_server, update_config),
- {ok, St};
-handle_config_change(_Sec, _Key, _Val, _Persist, St) ->
- {ok, St}.
-
-handle_config_terminate(_Server, _Reason, _State) ->
- gen_server:cast(ioq_server, update_config),
- ok.
+ {ok, { {one_for_one, 5, 10}, [?CHILD(ioq, worker)]}}.
[couchdb] 19/19: Remove ioq as dependency and add as subdir
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3c50283be8b20090b76826ef76917d58889fc915
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Mon Jan 6 13:26:28 2020 -0500
Remove ioq as dependency and add as subdir
---
rebar.config.script | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rebar.config.script b/rebar.config.script
index 79d3e0c..da6a89b 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -131,6 +131,7 @@ SubDirs = [
"src/dreyfus",
"src/fabric",
"src/global_changes",
+ "src/ioq",
"src/ken",
"src/mango",
"src/rexi",
@@ -146,7 +147,6 @@ DepDescs = [
{ets_lru, "ets-lru", {tag, "1.0.0"}},
{khash, "khash", {tag, "1.0.1"}},
{snappy, "snappy", {tag, "CouchDB-1.0.4"}},
-{ioq, "ioq", {tag, "2.1.2"}},
{hqueue, "hqueue", {tag, "1.0.1"}},
%% Non-Erlang deps
[couchdb] 05/19: Allow to customize concurrency value
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c552c665dc6af0ca750ce074c816c4fcb0f05962
Author: Alexander Shorin <kx...@apache.org>
AuthorDate: Mon Sep 22 17:35:37 2014 +0400
Allow to customize concurrency value
---
src/ioq.erl | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 9bfb1f8..4598c37 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -21,7 +21,7 @@
-export([handle_config_change/5]).
-record(state, {
- concurrency=10,
+ concurrency,
ratio,
interactive=queue:new(),
compaction=queue:new(),
@@ -55,7 +55,8 @@ init(_) ->
read_config(State) ->
Ratio = list_to_float(config:get("ioq", "ratio", "0.01")),
- State#state{ratio=Ratio}.
+ Concurrency = list_to_integer(config:get("ioq", "concurrency", "10")),
+ State#state{concurrency=Concurrency, ratio=Ratio}.
handle_call(#request{}=Request, From, State) ->
{noreply, enqueue_request(Request#request{from=From}, State), 0}.
[couchdb] 01/19: Initial commit
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6293a8b88d1cd67c254ac764a046a1c3a0905be4
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Jul 17 17:53:45 2014 +0100
Initial commit
This is substantively the work from branch 1775-feature-io-regulator
but with erlang application paraphenalia.
---
.gitignore | 2 +
src/ioq.app.src | 21 ++++++++++
src/ioq.erl | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/ioq_app.erl | 21 ++++++++++
src/ioq_sup.erl | 24 +++++++++++
5 files changed, 194 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..21cf3d3
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.rebar
+ebin/
diff --git a/src/ioq.app.src b/src/ioq.app.src
new file mode 100644
index 0000000..65ea50d
--- /dev/null
+++ b/src/ioq.app.src
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application,ioq, [
+ {description, "I/O prioritizing engine"},
+ {vsn, git},
+ {registered,[]},
+ {applications,[kernel,stdlib,config]},
+ {mod,{ioq_app,[]}},
+ {env, []},
+ {modules,[ioq,ioq_app,ioq_sup]}
+]}.
diff --git a/src/ioq.erl b/src/ioq.erl
new file mode 100644
index 0000000..686a696
--- /dev/null
+++ b/src/ioq.erl
@@ -0,0 +1,126 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ioq).
+-behaviour(gen_server).
+
+-export([start_link/0, call/2]).
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]).
+
+-record(state, {
+ concurrency=10,
+ ratio,
+ interactive=queue:new(),
+ compaction=queue:new(),
+ running=[]
+}).
+
+-record(request, {
+ fd,
+ msg,
+ class,
+ from,
+ ref
+}).
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+call(Fd, Msg) ->
+ Request = #request{fd=Fd, msg=Msg, class=get(io_class), from=self()},
+ gen_server:call(?MODULE, Request, infinity).
+
+init(_) ->
+ Ratio = list_to_float(config:get("ioq", "ratio", "0.01")),
+ {ok, #state{ratio=Ratio}}.
+
+handle_call(#request{}=Request, From, State) ->
+ {noreply, enqueue_request(Request#request{from=From}, State), 0}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info({Ref, Reply}, State) ->
+ case lists:keytake(Ref, #request.ref, State#state.running) of
+ {value, Request, Remaining} ->
+ erlang:demonitor(Ref, [flush]),
+ gen_server:reply(Request#request.from, Reply),
+ {noreply, State#state{running=Remaining}, 0};
+ false ->
+ {noreply, State, 0}
+ end;
+
+handle_info({'DOWN', Ref, _, _, Reason}, State) ->
+ case lists:keytake(Ref, #request.ref, State#state.running) of
+ {value, Request, Remaining} ->
+ gen_server:reply(Request#request.from, {'EXIT', Reason}),
+ {noreply, State#state{running=Remaining}, 0};
+ false ->
+ {noreply, State, 0}
+ end;
+
+handle_info(timeout, State) ->
+ {noreply, maybe_submit_request(State)}.
+
+code_change(_Vsn, State, _Extra) ->
+ {ok, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+enqueue_request(#request{class=compaction}=Request, #state{}=State) ->
+ State#state{compaction=queue:in(Request, State#state.compaction)};
+enqueue_request(#request{}=Request, #state{}=State) ->
+ State#state{interactive=queue:in(Request, State#state.interactive)}.
+
+maybe_submit_request(#state{concurrency=Concurrency, running=Running}=State)
+ when length(Running) < Concurrency ->
+ case make_next_request(State) of
+ State ->
+ State;
+ NewState when length(Running) >= Concurrency - 1 ->
+ NewState;
+ NewState ->
+ maybe_submit_request(NewState)
+ end;
+maybe_submit_request(State) ->
+ State.
+
+make_next_request(#state{}=State) ->
+ case {queue:is_empty(State#state.compaction), queue:is_empty(State#state.interactive)} of
+ {true, true} ->
+ State;
+ {true, false} ->
+ choose_next_request(#state.interactive, State);
+ {false, true} ->
+ choose_next_request(#state.compaction, State);
+ {false, false} ->
+ case random:uniform() < State#state.ratio of
+ true ->
+ choose_next_request(#state.compaction, State);
+ false ->
+ choose_next_request(#state.interactive, State)
+ end
+ end.
+
+choose_next_request(Index, State) ->
+ case queue:out(element(Index, State)) of
+ {empty, _} ->
+ State;
+ {{value, Request}, Q} ->
+ submit_request(Request, setelement(Index, State, Q))
+ end.
+
+submit_request(#request{}=Request, #state{}=State) ->
+ Ref = erlang:monitor(process, Request#request.fd),
+ Request#request.fd ! {'$gen_call', {self(), Ref}, Request#request.msg},
+ State#state{running = [Request#request{ref=Ref} | State#state.running]}.
diff --git a/src/ioq_app.erl b/src/ioq_app.erl
new file mode 100644
index 0000000..2e6d75a
--- /dev/null
+++ b/src/ioq_app.erl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ioq_app).
+-behaviour(application).
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+ ioq_sup:start_link().
+
+stop(_State) ->
+ ok.
diff --git a/src/ioq_sup.erl b/src/ioq_sup.erl
new file mode 100644
index 0000000..c4d04a9
--- /dev/null
+++ b/src/ioq_sup.erl
@@ -0,0 +1,24 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ioq_sup).
+-behaviour(supervisor).
+-export([start_link/0, init/1]).
+
+%% Helper macro for declaring children of supervisor
+-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+init([]) ->
+ {ok, { {one_for_one, 5, 10}, [?CHILD(ioq, worker)]}}.
[couchdb] 12/19: Merge remote branch
'cloudant:3102-fix-config_subscription'
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ba99ec70d31cf82fe4868ab4bb92b0978f4fe67f
Merge: c7c75eb 126a849
Author: ILYA Khlopotov <ii...@ca.ibm.com>
AuthorDate: Tue Aug 23 14:59:51 2016 -0700
Merge remote branch 'cloudant:3102-fix-config_subscription'
This closes #4
Signed-off-by: ILYA Khlopotov <ii...@ca.ibm.com>
src/ioq.erl | 16 +++++++++-------
src/ioq_sup.erl | 23 ++++++++++++++++++++++-
2 files changed, 31 insertions(+), 8 deletions(-)
[couchdb] 08/19: Don't restart event handler on termination
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0ffa7cd9fd1e89ae667ed234d21e04696e3033ae
Author: ILYA Khlopotov <ii...@ca.ibm.com>
AuthorDate: Fri Jan 30 11:30:02 2015 -0800
Don't restart event handler on termination
COUCHDB-2561
---
src/ioq.erl | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/ioq.erl b/src/ioq.erl
index c4b3b4e..967a49b 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -91,6 +91,7 @@ handle_config_change("ioq", _, _, _, _) ->
handle_config_change(_, _, _, _, _) ->
{ok, nil}.
+handle_config_terminate(_, stop, _) -> ok;
handle_config_terminate(_, _, _) ->
spawn(fun() ->
timer:sleep(5000),
[couchdb] 09/19: Merge remote-tracking branch
'iilyak/2561-make-config-API-consistent'
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c7c75ebeaf41599e3a3e211097d864f0e7785829
Merge: 2175035 0ffa7cd
Author: Alexander Shorin <kx...@apache.org>
AuthorDate: Wed Feb 4 18:43:21 2015 +0300
Merge remote-tracking branch 'iilyak/2561-make-config-API-consistent'
This closes #3
COUCHDB-2561
src/ioq.erl | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
[couchdb] 04/19: Update state on config changes
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 40d157f39c0fa0d80db9ccf61b770b72103ddbed
Author: Alexander Shorin <kx...@apache.org>
AuthorDate: Mon Sep 22 17:32:44 2014 +0400
Update state on config changes
---
src/ioq.erl | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index b761a0b..9bfb1f8 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -12,10 +12,14 @@
-module(ioq).
-behaviour(gen_server).
+-behaviour(config_listener).
-export([start_link/0, call/3]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]).
+% config_listener api
+-export([handle_config_change/5]).
+
-record(state, {
concurrency=10,
ratio,
@@ -45,12 +49,19 @@ call(Fd, Msg, Priority) ->
end.
init(_) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ State = #state{},
+ {ok, read_config(State)}.
+
+read_config(State) ->
Ratio = list_to_float(config:get("ioq", "ratio", "0.01")),
- {ok, #state{ratio=Ratio}}.
+ State#state{ratio=Ratio}.
handle_call(#request{}=Request, From, State) ->
{noreply, enqueue_request(Request#request{from=From}, State), 0}.
+handle_cast(change, State) ->
+ {noreply, read_config(State)};
handle_cast(_Msg, State) ->
{noreply, State}.
@@ -76,6 +87,11 @@ handle_info({'DOWN', Ref, _, _, Reason}, State) ->
handle_info(timeout, State) ->
{noreply, maybe_submit_request(State)}.
+handle_config_change("ioq", _, _, _, _) ->
+ {ok, gen_server:cast(?MODULE, change)};
+handle_config_change(_, _, _, _, _) ->
+ {ok, nil}.
+
code_change(_Vsn, State, _Extra) ->
{ok, State}.
[couchdb] 15/19: Use couch_rand compatibility module
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 345804ce4d34786acbf0f498a93eac7013a2b0b5
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu Oct 5 12:05:28 2017 -0400
Use couch_rand compatibility module
---
src/ioq.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 93377d6..9ca2656 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -136,7 +136,7 @@ make_next_request(#state{}=State) ->
{false, true} ->
choose_next_request(#state.compaction, State);
{false, false} ->
- case random:uniform() < State#state.ratio of
+ case couch_rand:uniform() < State#state.ratio of
true ->
choose_next_request(#state.compaction, State);
false ->
[couchdb] 10/19: Update handle_config_terminate API
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bdcfe6aa3a33c28411f0e4b9b121fff33474c3eb
Author: ILYA Khlopotov <ii...@ca.ibm.com>
AuthorDate: Mon Aug 22 14:42:07 2016 -0700
Update handle_config_terminate API
COUCHDB-3102
---
src/ioq.erl | 16 +++++++++-------
src/ioq_sup.erl | 23 ++++++++++++++++++++++-
2 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 967a49b..93377d6 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -20,6 +20,8 @@
% config_listener api
-export([handle_config_change/5, handle_config_terminate/3]).
+-define(RELISTEN_DELAY, 5000).
+
-record(state, {
concurrency,
ratio,
@@ -83,6 +85,9 @@ handle_info({'DOWN', Ref, _, _, Reason}, State) ->
false ->
{noreply, State, 0}
end;
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
handle_info(timeout, State) ->
{noreply, maybe_submit_request(State)}.
@@ -91,13 +96,10 @@ handle_config_change("ioq", _, _, _, _) ->
handle_config_change(_, _, _, _, _) ->
{ok, nil}.
-handle_config_terminate(_, stop, _) -> ok;
-handle_config_terminate(_, _, _) ->
- spawn(fun() ->
- timer:sleep(5000),
- config:listen_for_changes(?MODULE, nil)
- end),
- ok.
+handle_config_terminate(_Server, stop, _State) ->
+ ok;
+handle_config_terminate(_Server, _Reason, _State) ->
+ erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), restart_config_listener).
code_change(_Vsn, State, _Extra) ->
{ok, State}.
diff --git a/src/ioq_sup.erl b/src/ioq_sup.erl
index c4d04a9..56e51ae 100644
--- a/src/ioq_sup.erl
+++ b/src/ioq_sup.erl
@@ -13,6 +13,7 @@
-module(ioq_sup).
-behaviour(supervisor).
-export([start_link/0, init/1]).
+-export([handle_config_change/5, handle_config_terminate/3]).
%% Helper macro for declaring children of supervisor
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
@@ -21,4 +22,24 @@ start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->
- {ok, { {one_for_one, 5, 10}, [?CHILD(ioq, worker)]}}.
+ {ok, { {one_for_one, 5, 10}, [
+ {
+ config_listener_mon,
+ {config_listener_mon, start_link, [?MODULE, nil]},
+ permanent,
+ 5000,
+ worker,
+ [config_listener_mon]
+ },
+ ?CHILD(ioq_server, worker)
+ ]} }.
+
+handle_config_change("ioq", _Key, _Val, _Persist, St) ->
+ gen_server:cast(ioq_server, update_config),
+ {ok, St};
+handle_config_change(_Sec, _Key, _Val, _Persist, St) ->
+ {ok, St}.
+
+handle_config_terminate(_Server, _Reason, _State) ->
+ gen_server:cast(ioq_server, update_config),
+ ok.
[couchdb] 16/19: Merge branch 'use-couch-rand-module'
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 04bebb3f99b2045da0deb090afb4e6c4eacb67f5
Merge: 1d2b149 345804c
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu Oct 5 13:22:37 2017 -0400
Merge branch 'use-couch-rand-module'
Fixes #6
src/ioq.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[couchdb] 06/19: Handle {gen_event_EXIT, {config_listener, ioq},
shutdown} message
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 217503577e23f26dfd2f9cbaa52e9f78aaa3b308
Author: Alexander Shorin <kx...@apache.org>
AuthorDate: Tue Jan 13 03:28:39 2015 +0300
Handle {gen_event_EXIT,{config_listener,ioq},shutdown} message
---
src/ioq.erl | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 4598c37..6c01b9c 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -75,7 +75,6 @@ handle_info({Ref, Reply}, State) ->
false ->
{noreply, State, 0}
end;
-
handle_info({'DOWN', Ref, _, _, Reason}, State) ->
case lists:keytake(Ref, #request.ref, State#state.running) of
{value, Request, Remaining} ->
@@ -84,7 +83,12 @@ handle_info({'DOWN', Ref, _, _, Reason}, State) ->
false ->
{noreply, State, 0}
end;
-
+handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
+ erlang:send_after(5000, self(), restart_config_listener),
+ {noreply, State};
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, nil),
+ {noreply, State};
handle_info(timeout, State) ->
{noreply, maybe_submit_request(State)}.
[couchdb] 02/19: new IOQ api
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch ioq-in-tree
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 04aea732058c5f0b7081b1fa444dbf4b3761cd02
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Sep 1 19:20:09 2014 +0100
new IOQ api
---
src/ioq.erl | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/src/ioq.erl b/src/ioq.erl
index 686a696..4983b73 100644
--- a/src/ioq.erl
+++ b/src/ioq.erl
@@ -13,7 +13,7 @@
-module(ioq).
-behaviour(gen_server).
--export([start_link/0, call/2]).
+-export([start_link/0, call/3]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3, terminate/2]).
-record(state, {
@@ -27,7 +27,7 @@
-record(request, {
fd,
msg,
- class,
+ priority,
from,
ref
}).
@@ -35,8 +35,8 @@
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-call(Fd, Msg) ->
- Request = #request{fd=Fd, msg=Msg, class=get(io_class), from=self()},
+call(Fd, Msg, Priority) ->
+ Request = #request{fd=Fd, msg=Msg, priority=Priority, from=self()},
gen_server:call(?MODULE, Request, infinity).
init(_) ->
@@ -77,7 +77,9 @@ code_change(_Vsn, State, _Extra) ->
terminate(_Reason, _State) ->
ok.
-enqueue_request(#request{class=compaction}=Request, #state{}=State) ->
+enqueue_request(#request{priority={db_compact, _}}=Request, #state{}=State) ->
+ State#state{compaction=queue:in(Request, State#state.compaction)};
+enqueue_request(#request{priority={view_compact, _, _}}=Request, #state{}=State) ->
State#state{compaction=queue:in(Request, State#state.compaction)};
enqueue_request(#request{}=Request, #state{}=State) ->
State#state{interactive=queue:in(Request, State#state.interactive)}.