You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2021/09/10 00:22:32 UTC
[couchdb] branch chewbranca-ioq-experiments-rebase updated: Use the
same IOQ Pid for views and compaction
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch chewbranca-ioq-experiments-rebase
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/chewbranca-ioq-experiments-rebase by this push:
new 67d6e0a Use the same IOQ Pid for views and compaction
67d6e0a is described below
commit 67d6e0ad39fd7f5891b2709fbc2aefecacf4d283
Author: Russell Branca <ch...@apache.org>
AuthorDate: Thu Sep 9 17:12:54 2021 -0700
Use the same IOQ Pid for views and compaction
---
src/couch/src/couch_bt_engine.erl | 19 +++++++++++++++----
src/couch/src/couch_bt_engine_compactor.erl | 14 ++++++++------
src/couch/src/couch_db.erl | 4 ++++
src/couch/src/couch_db_engine.erl | 10 ++++++++++
src/couch/src/couch_file.erl | 13 +++++++++++--
src/couch_mrview/src/couch_mrview_compactor.erl | 3 ++-
src/couch_mrview/src/couch_mrview_index.erl | 3 ++-
src/couch_mrview/src/couch_mrview_util.erl | 8 ++++----
8 files changed, 56 insertions(+), 18 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index dc5bbfc..e081570 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -20,6 +20,7 @@
delete_compaction_files/3,
init/2,
+ init/3,
terminate/2,
handle_db_updater_call/2,
handle_db_updater_info/2,
@@ -30,6 +31,7 @@
last_activity/1,
+ get_fd_handle/1,
get_compacted_seq/1,
get_del_doc_count/1,
get_disk_version/1,
@@ -146,7 +148,11 @@ delete_compaction_files(RootDir, FilePath, DelOpts) ->
init(FilePath, Options) ->
- {ok, Fd} = open_db_file(FilePath, Options),
+ init(FilePath, Options, undefined).
+
+
+init(FilePath, Options, IOQPid) ->
+ {ok, Fd} = open_db_file(FilePath, Options, IOQPid),
Header = case lists:member(create, Options) of
true ->
delete_compaction_files(FilePath),
@@ -215,6 +221,10 @@ last_activity(#st{fd = Fd}) ->
couch_file:last_read(Fd).
+get_fd_handle(#st{fd = Fd}) ->
+ Fd.
+
+
get_compacted_seq(#st{header = Header}) ->
couch_bt_engine_header:get(Header, compacted_seq).
@@ -655,7 +665,8 @@ start_compaction(St, DbName, Options, Parent) ->
finish_compaction(OldState, DbName, Options, CompactFilePath) ->
- {ok, NewState1} = ?MODULE:init(CompactFilePath, Options),
+ IOQPid = ioq:ioq_pid(OldState#st.fd),
+ {ok, NewState1} = ?MODULE:init(CompactFilePath, Options, IOQPid),
OldSeq = get_update_seq(OldState),
NewSeq = get_update_seq(NewState1),
case OldSeq == NewSeq of
@@ -846,10 +857,10 @@ copy_props(#st{header = Header} = St, Props) ->
}}.
-open_db_file(FilePath, Options) ->
+open_db_file(FilePath, Options, IOQPid) ->
Hash = list_to_atom(integer_to_list(mem3_hash:crc32(FilePath))),
erlang:put(couch_file_hash, Hash),
- case couch_file:open(FilePath, Options) of
+ case couch_file:open(FilePath, Options, IOQPid) of
{ok, Fd} ->
{ok, Fd};
{error, enoent} ->
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 297e70c..40457f5 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -92,12 +92,14 @@ start(#st{} = St, DbName, Options, Parent) ->
open_compaction_files(DbName, OldSt, Options) ->
#st{
filepath = DbFilePath,
- header = SrcHdr
+ header = SrcHdr,
+ fd = OldFd
} = OldSt,
+ IOQPid = ioq:ioq_pid(OldFd),
DataFile = DbFilePath ++ ".compact.data",
MetaFile = DbFilePath ++ ".compact.meta",
- {ok, DataFd, DataHdr} = open_compaction_file(DataFile),
- {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile),
+ {ok, DataFd, DataHdr} = open_compaction_file(DataFile, IOQPid),
+ {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile, IOQPid),
DataHdrIsDbHdr = couch_bt_engine_header:is_header(DataHdr),
CompSt = case {DataHdr, MetaHdr} of
{#comp_header{}=A, #comp_header{}=A} ->
@@ -577,15 +579,15 @@ compact_final_sync(#comp_st{new_st = St0} = CompSt) ->
}.
-open_compaction_file(FilePath) ->
- case couch_file:open(FilePath, [nologifmissing]) of
+open_compaction_file(FilePath, IOQPid) ->
+ case couch_file:open(FilePath, [nologifmissing], IOQPid) of
{ok, Fd} ->
case couch_file:read_header(Fd) of
{ok, Header} -> {ok, Fd, Header};
no_valid_header -> {ok, Fd, nil}
end;
{error, enoent} ->
- {ok, Fd} = couch_file:open(FilePath, [create]),
+ {ok, Fd} = couch_file:open(FilePath, [create], IOQPid),
{ok, Fd, nil}
end.
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index aec0f54..256d943 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -45,6 +45,7 @@
get_epochs/1,
get_filepath/1,
get_instance_start_time/1,
+ get_fd_handle/1,
get_pid/1,
get_revs_limit/1,
get_security/1,
@@ -207,6 +208,9 @@ clustered_db(DbName, #user_ctx{} = UserCtx) ->
clustered_db(DbName, UserCtx, SecProps) ->
clustered_db(DbName, [{user_ctx, UserCtx}, {security, SecProps}]).
+get_fd_handle(#db{} = Db) ->
+ couch_db_engine:get_fd_handle(Db).
+
is_db(#db{}) ->
true;
is_db(_) ->
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index f1ba81c..fcfa6f2 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -193,6 +193,10 @@
% All of the get_* functions may be called from many
% processes concurrently.
+% Get the FD handle, usually for #ioq_file{}
+-callback get_fd_handle(DbHandle::db_handle()) -> any().
+
+
% The database should make a note of the update sequence when it
% was last compacted. If the database doesn't need compacting it
% can just hard code a return value of 0.
@@ -707,6 +711,7 @@
last_activity/1,
get_engine/1,
+ get_fd_handle/1,
get_compacted_seq/1,
get_del_doc_count/1,
get_disk_version/1,
@@ -839,6 +844,11 @@ get_engine(#db{} = Db) ->
Engine.
+get_fd_handle(#db{} = Db) ->
+ #db{engine = {Engine, EngineState}} = Db,
+ Engine:get_fd_handle(EngineState).
+
+
get_compacted_seq(#db{} = Db) ->
#db{engine = {Engine, EngineState}} = Db,
Engine:get_compacted_seq(EngineState).
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index a3856b9..e63b583 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -39,7 +39,7 @@
}).
% public API
--export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]).
+-export([open/1, open/2, open/3, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]).
-export([pread_term/2, pread_iolist/2, pread_binary/2]).
-export([append_binary/2, append_binary_md5/2]).
-export([append_raw_chunk/2, assemble_file_chunk/1, assemble_file_chunk/2]).
@@ -68,10 +68,19 @@ open(Filepath) ->
open(Filepath, []).
open(Filepath, Options) ->
+ open(Filepath, Options, undefined).
+
+open(Filepath, Options, IOQPid0) ->
case gen_server:start_link(couch_file,
{Filepath, Options, self(), Ref = make_ref()}, []) of
{ok, Fd} ->
- {ok, IOQPid} = ioq_server2:start_link({by_shard, Filepath, Fd}),
+ IOQPid = case IOQPid0 of
+ undefined ->
+ {ok, IOQPid1} = ioq_server2:start_link({by_shard, Filepath, Fd}),
+ IOQPid1;
+ IOQPid0 when is_pid(IOQPid0) ->
+ IOQPid0
+ end,
Tab = gen_server:call(Fd, get_cache_ref),
{ok, #ioq_file{fd=Fd, ioq=IOQPid, tab=Tab}};
ignore ->
diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl
index f6b5e11..100df88 100644
--- a/src/couch_mrview/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview/src/couch_mrview_compactor.erl
@@ -47,8 +47,9 @@ compact(State) ->
erlang:put(io_priority, {view_compact, DbName, IdxName}),
{EmptyState, NumDocIds} = couch_util:with_db(DbName, fun(Db) ->
+ IOQPid = ioq:ioq_pid(couch_db:get_fd_handle(Db)),
CompactFName = couch_mrview_util:compaction_file(DbName, Sig),
- {ok, Fd} = couch_mrview_util:open_file(CompactFName),
+ {ok, Fd} = couch_mrview_util:open_file(CompactFName, IOQPid),
ESt = couch_mrview_util:reset_index(Db, Fd, State),
{ok, Count} = couch_db:get_doc_count(Db),
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 582595c..f0c0db7 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -92,6 +92,7 @@ open(Db, State0) ->
db_name=DbName,
sig=Sig
} = State = set_partitioned(Db, State0),
+ IOQPid = ioq:ioq_pid(couch_db:get_fd_handle(Db)),
IndexFName = couch_mrview_util:index_file(DbName, Sig),
% If we are upgrading from <= 2.x, we upgrade the view
@@ -111,7 +112,7 @@ open(Db, State0) ->
OldSig = couch_mrview_util:maybe_update_index_file(State),
- case couch_mrview_util:open_file(IndexFName) of
+ case couch_mrview_util:open_file(IndexFName, IOQPid) of
{ok, Fd} ->
case couch_file:read_header(Fd) of
% upgrade code for <= 2.x
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 6486c93..dfb6608 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -17,7 +17,7 @@
-export([verify_view_filename/1, get_signature_from_filename/1]).
-export([ddoc_to_mrst/2, init_state/4, reset_index/3]).
-export([make_header/1]).
--export([index_file/2, compaction_file/2, open_file/1]).
+-export([index_file/2, compaction_file/2, open_file/2]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
@@ -736,10 +736,10 @@ compaction_file(DbName, Sig) ->
couch_index_util:index_file(mrview, DbName, FileName).
-open_file(FName) ->
- case couch_file:open(FName, [nologifmissing]) of
+open_file(FName, IOQPid) ->
+ case couch_file:open(FName, [nologifmissing], IOQPid) of
{ok, Fd} -> {ok, Fd};
- {error, enoent} -> couch_file:open(FName, [create]);
+ {error, enoent} -> couch_file:open(FName, [create], IOQPid);
Error -> Error
end.