You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2022/08/08 22:48:36 UTC
[couchdb] branch cost-counting created (now 94341d283)
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a change to branch cost-counting
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at 94341d283 Updated PoC of cost accounting
This branch includes the following new commits:
new 94dfaac5a Cost counting hackery
new d9c0e8942 Add btree cost
new 94341d283 Updated PoC of cost accounting
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 02/03: Add btree cost
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch cost-counting
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d9c0e894243ca4ef9dec988979ccf714cf6fc6eb
Author: Russell Branca <ch...@apache.org>
AuthorDate: Thu Jul 21 13:03:28 2022 -0700
Add btree cost
---
src/couch/src/couch_btree.erl | 1 +
src/couch/src/couch_cost.erl | 22 ++++++++++++++++++++--
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index b974a22ee..2c75287ee 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -1077,6 +1077,7 @@ stream_node(Bt, Reds, Node, StartKey, InRange, Dir, Fun, Acc) ->
stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc) ->
Pointer = element(1, Node),
{NodeType, NodeList} = get_node(Bt, Pointer),
+ couch_cost:inc_get_node(NodeType),
case NodeType of
kp_node ->
stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc);
diff --git a/src/couch/src/couch_cost.erl b/src/couch/src/couch_cost.erl
index b37f34381..25929501d 100644
--- a/src/couch/src/couch_cost.erl
+++ b/src/couch/src/couch_cost.erl
@@ -2,7 +2,8 @@
-export([
inc_doc/0, inc_doc/1, inc_doc/2,
- inc_ioq/0, inc_ioq/1, inc_ioq/2%%,
+ inc_ioq/0, inc_ioq/1, inc_ioq/2,
+ inc_get_node/1
%%io_bytes_read/1, io_bytes_read/2,
%%io_bytes_written/1, io_bytes_written/2,
%%inc_js_evals/0, inc_js_evals/1, inc_js_evals/2
@@ -17,7 +18,9 @@
ioq_calls = 0,
io_bytes_read = 0,
io_bytes_written = 0,
- js_evals = 0
+ js_evals = 0,
+ get_kv_node = 0,
+ get_kp_node = 0
}).
get_cost() ->
@@ -40,3 +43,18 @@ inc_doc(N, #cost{docs_read=DR0}=Cost) -> update_cost(Cost#cost{docs_read=DR0+N})
inc_ioq() -> inc_ioq(1).
inc_ioq(N) -> inc_ioq(N, get_cost()).
inc_ioq(N, #cost{ioq_calls=IOQ0}=Cost) -> update_cost(Cost#cost{ioq_calls=IOQ0+N}).
+
+
+inc_get_node(Type) when Type =:= kp_node orelse Type =:= kv_node ->
+%%inc_get_node(Type) when Type =:= kp_node; Type =:= kv_node ->
+ update_cost(inc(Type, get_cost())).
+
+
+inc(Key, Cost) ->
+ inc(Key, Cost, 1).
+
+
+inc(kp_node, #cost{get_kp_node=GKP}=Cost, N) ->
+ Cost#cost{get_kp_node = GKP + N};
+inc(kv_node, #cost{get_kv_node=GKV}=Cost, N) ->
+ Cost#cost{get_kp_node = GKV + N}.
[couchdb] 03/03: Updated PoC of cost accounting
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch cost-counting
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 94341d283c4bad1951d92edd4b970c16cfba4eed
Author: Russell Branca <ch...@apache.org>
AuthorDate: Mon Aug 8 15:48:24 2022 -0700
Updated PoC of cost accounting
---
src/couch/src/couch_btree.erl | 3 +-
src/couch/src/couch_cost.erl | 58 +++++++++++++++++++++++++++++++++--
src/couch/src/couch_query_servers.erl | 2 ++
src/couch/src/couch_server.erl | 1 +
src/fabric/src/fabric_util.erl | 8 +++--
src/rexi/src/rexi_utils.erl | 10 +++---
6 files changed, 72 insertions(+), 10 deletions(-)
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index 2c75287ee..2d64ea4ba 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -472,6 +472,7 @@ reduce_tree_size(kp_node, NodeSize, [{_K, {_P, _Red, Sz}} | NodeList]) ->
get_node(#btree{fd = Fd}, NodePos) ->
{ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos),
+ couch_cost:inc_get_node(NodeType),
{NodeType, NodeList}.
write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) ->
@@ -1077,7 +1078,6 @@ stream_node(Bt, Reds, Node, StartKey, InRange, Dir, Fun, Acc) ->
stream_node(Bt, Reds, Node, InRange, Dir, Fun, Acc) ->
Pointer = element(1, Node),
{NodeType, NodeList} = get_node(Bt, Pointer),
- couch_cost:inc_get_node(NodeType),
case NodeType of
kp_node ->
stream_kp_node(Bt, Reds, adjust_dir(Dir, NodeList), InRange, Dir, Fun, Acc);
@@ -1164,6 +1164,7 @@ stream_kv_node2(Bt, Reds, PrevKVs, [{K, V} | RestKVs], InRange, Dir, Fun, Acc) -
false ->
{stop, {PrevKVs, Reds}, Acc};
true ->
+ couch_cost:inc_changes_processed(),
AssembledKV = assemble(Bt, K, V),
case Fun(visit, AssembledKV, {PrevKVs, Reds}, Acc) of
{ok, Acc2} ->
diff --git a/src/couch/src/couch_cost.erl b/src/couch/src/couch_cost.erl
index 25929501d..ebb25cf8f 100644
--- a/src/couch/src/couch_cost.erl
+++ b/src/couch/src/couch_cost.erl
@@ -3,22 +3,33 @@
-export([
inc_doc/0, inc_doc/1, inc_doc/2,
inc_ioq/0, inc_ioq/1, inc_ioq/2,
- inc_get_node/1
+ inc_get_node/1,
+ inc_db_open/0,
+ inc_js_filter/0, inc_js_filter/1, inc_js_filter/2,
+ inc_js_filtered_docs/1, inc_js_filtered_docs/2,
+ inc_changes_processed/0, inc_changes_processed/1, inc_changes_processed/2
%%io_bytes_read/1, io_bytes_read/2,
%%io_bytes_written/1, io_bytes_written/2,
%%inc_js_evals/0, inc_js_evals/1, inc_js_evals/2
]).
-export([
- get_cost/0
+ get_cost/0,
+ get_costs/0,
+ accumulate_costs/1
]).
+
-record(cost, {
+ db_open = 0,
docs_read = 0,
+ changes_processed = 0,
ioq_calls = 0,
io_bytes_read = 0,
io_bytes_written = 0,
js_evals = 0,
+ js_filter = 0,
+ js_filtered_docs = 0,
get_kv_node = 0,
get_kp_node = 0
}).
@@ -33,9 +44,23 @@ get_cost() ->
Cost
end.
+get_costs() ->
+ case get(cost_accounting_context_aggregation) of
+ undefined ->
+ Costs = [],
+ put(cost_accounting_context_aggregation, Costs),
+ Costs;
+ Costs when is_list(Costs) ->
+ Costs
+ end.
+
update_cost(#cost{}=Cost) ->
put(cost_accounting_context, Cost).
+accumulate_costs(#cost{}=Cost) ->
+ Costs = get_costs(),
+ put(cost_accounting_context_aggregation, [Cost | Costs]).
+
inc_doc() -> inc_doc(1).
inc_doc(N) -> inc_doc(N, get_cost()).
inc_doc(N, #cost{docs_read=DR0}=Cost) -> update_cost(Cost#cost{docs_read=DR0+N}).
@@ -50,6 +75,25 @@ inc_get_node(Type) when Type =:= kp_node orelse Type =:= kv_node ->
update_cost(inc(Type, get_cost())).
+inc_js_filter() -> inc_js_filter(1).
+inc_js_filter(N) -> inc_js_filter(N, get_cost()).
+inc_js_filter(N, #cost{}=Cost) -> update_cost(inc(js_filter, Cost, N)).
+
+
+inc_js_filtered_docs(N) -> inc_js_filtered_docs(N, get_cost()).
+inc_js_filtered_docs(N, #cost{}=Cost) -> update_cost(inc(js_filtered_docs, Cost, N)).
+
+
+inc_changes_processed() -> inc_changes_processed(1).
+inc_changes_processed(N) -> inc_changes_processed(N, get_cost()).
+inc_changes_processed(N, #cost{}=Cost) -> update_cost(inc(changes_processed, Cost, N)).
+
+
+inc_db_open() -> inc_db_open(1).
+inc_db_open(N) -> inc_db_open(N, get_cost()).
+inc_db_open(N, #cost{}=Cost) -> update_cost(inc(db_open, Cost, N)).
+
+
inc(Key, Cost) ->
inc(Key, Cost, 1).
@@ -57,4 +101,12 @@ inc(Key, Cost) ->
inc(kp_node, #cost{get_kp_node=GKP}=Cost, N) ->
Cost#cost{get_kp_node = GKP + N};
inc(kv_node, #cost{get_kv_node=GKV}=Cost, N) ->
- Cost#cost{get_kp_node = GKV + N}.
+ Cost#cost{get_kv_node = GKV + N};
+inc(changes_processed, #cost{changes_processed=CP}=Cost, N) ->
+ Cost#cost{changes_processed = CP + N};
+inc(db_open, #cost{db_open=DBO}=Cost, N) ->
+ Cost#cost{db_open = DBO + N};
+inc(js_filter, #cost{js_filter=JSF}=Cost, N) ->
+ Cost#cost{js_filter = JSF + N};
+inc(js_filtered_docs, #cost{js_filtered_docs=JSFD}=Cost, N) ->
+ Cost#cost{js_filtered_docs = JSFD + N}.
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
index cb619405e..641e83275 100644
--- a/src/couch/src/couch_query_servers.erl
+++ b/src/couch/src/couch_query_servers.erl
@@ -527,6 +527,8 @@ filter_view(DDoc, VName, Docs) ->
{ok, Passes}.
filter_docs(Req, Db, DDoc, FName, Docs) ->
+ couch_cost:inc_js_filter(),
+ couch_cost:inc_js_filtered_docs(length(Docs)),
JsonReq =
case Req of
{json_req, JsonObj} ->
diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl
index 23fdd584b..4b4c213f9 100644
--- a/src/couch/src/couch_server.erl
+++ b/src/couch/src/couch_server.erl
@@ -96,6 +96,7 @@ sup_start_link(N) ->
gen_server:start_link({local, couch_server(N)}, couch_server, [N], []).
open(DbName, Options) ->
+ couch_cost:inc_db_open(),
try
validate_open_or_create(DbName, Options),
open_int(DbName, Options)
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index 34e095403..3715fa7b0 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -138,7 +138,8 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) ->
Ref = rexi:cast(Node, self(), MFA, [sync]),
try
receive
- {Ref, {ok, Db}} ->
+ {Ref, {ok, Db}, {cost, Cost}} ->
+ couch_cost:accumulate_costs(Cost),
{ok, Db};
{Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} ->
throw(Error);
@@ -146,7 +147,10 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) ->
throw(Error);
{Ref, Reason} ->
couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]),
- get_shard(Rest, Opts, Timeout, Factor)
+ get_shard(Rest, Opts, Timeout, Factor);
+ Other ->
+ io:format("GOT UNEXPECTED MESSAGE FORMAT: ~p~n", [Other]),
+ erlang:error(Other)
after Timeout ->
couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]),
get_shard(Rest, Opts, Factor * Timeout, Factor)
diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl
index ae05bf7a6..b0bb79ae5 100644
--- a/src/rexi/src/rexi_utils.erl
+++ b/src/rexi/src/rexi_utils.erl
@@ -84,7 +84,8 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
{rexi, '$rexi_ping'} ->
{ok, Acc0};
{Ref, Msg, {cost,Cost}} ->
- io:format("GOT COST: ~p -- ~p~n", [Cost, Msg]),
+ io:format("[~p]GOT COST: ~p -- ~p~n", [self(), Cost, Msg]),
+ couch_cost:accumulate_costs(Cost),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
% this was some non-matching message which we will ignore
@@ -94,7 +95,8 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
end;
{Ref, From, Msg, {cost,Cost}} ->
%%io:format("GOT COST: ~p~n", [Cost]),
- io:format("GOT COST: ~p -- ~p~n", [Cost, Msg]),
+ io:format("[~p]GOT COST: ~p -- ~p~n", [self(), Cost, Msg]),
+ couch_cost:accumulate_costs(Cost),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
{ok, Acc0};
@@ -102,7 +104,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
Fun(Msg, {Worker, From}, Acc0)
end;
{Ref, Msg} ->
- io:format("GOT NON COST MSG: ~p~n", [Msg]),
+ %%io:format("GOT NON COST MSG: ~p~n", [Msg]),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
% this was some non-matching message which we will ignore
@@ -111,7 +113,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
Fun(Msg, Worker, Acc0)
end;
{Ref, From, Msg} ->
- io:format("GOT NON COST MSG: ~p~n", [Msg]),
+ %%io:format("GOT NON COST MSG: ~p~n", [Msg]),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
{ok, Acc0};
[couchdb] 01/03: Cost counting hackery
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chewbranca pushed a commit to branch cost-counting
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 94dfaac5a7d09a39098f3565d8ec5da26d768c62
Author: Russell Branca <ch...@apache.org>
AuthorDate: Mon Jul 18 16:26:00 2022 -0700
Cost counting hackery
---
src/couch/src/couch_cost.erl | 42 ++++++++++++++++++++++++++++++++++++++++++
src/couch/src/couch_db.erl | 1 +
src/fabric/src/fabric_rpc.erl | 2 ++
src/rexi/src/rexi.erl | 6 ++++--
src/rexi/src/rexi_utils.erl | 20 ++++++++++++++++++++
5 files changed, 69 insertions(+), 2 deletions(-)
diff --git a/src/couch/src/couch_cost.erl b/src/couch/src/couch_cost.erl
new file mode 100644
index 000000000..b37f34381
--- /dev/null
+++ b/src/couch/src/couch_cost.erl
@@ -0,0 +1,42 @@
+-module(couch_cost).
+
+-export([
+ inc_doc/0, inc_doc/1, inc_doc/2,
+ inc_ioq/0, inc_ioq/1, inc_ioq/2%%,
+ %%io_bytes_read/1, io_bytes_read/2,
+ %%io_bytes_written/1, io_bytes_written/2,
+ %%inc_js_evals/0, inc_js_evals/1, inc_js_evals/2
+]).
+
+-export([
+ get_cost/0
+]).
+
+-record(cost, {
+ docs_read = 0,
+ ioq_calls = 0,
+ io_bytes_read = 0,
+ io_bytes_written = 0,
+ js_evals = 0
+}).
+
+get_cost() ->
+ case get(cost_accounting_context) of
+ undefined ->
+ Cost = #cost{},
+ update_cost(Cost),
+ Cost;
+ #cost{}=Cost ->
+ Cost
+ end.
+
+update_cost(#cost{}=Cost) ->
+ put(cost_accounting_context, Cost).
+
+inc_doc() -> inc_doc(1).
+inc_doc(N) -> inc_doc(N, get_cost()).
+inc_doc(N, #cost{docs_read=DR0}=Cost) -> update_cost(Cost#cost{docs_read=DR0+N}).
+
+inc_ioq() -> inc_ioq(1).
+inc_ioq(N) -> inc_ioq(N, get_cost()).
+inc_ioq(N, #cost{ioq_calls=IOQ0}=Cost) -> update_cost(Cost#cost{ioq_calls=IOQ0+N}).
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index c7f1c8b5f..13c8894be 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -295,6 +295,7 @@ open_doc(Db, IdOrDocInfo) ->
open_doc(Db, Id, Options) ->
increment_stat(Db, [couchdb, database_reads]),
+ couch_cost:inc_doc(),
case open_doc_int(Db, Id, Options) of
{ok, #doc{deleted = true} = Doc} ->
case lists:member(deleted, Options) of
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 5db84458a..253294280 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -491,6 +491,8 @@ view_cb({meta, Meta}, Acc) ->
ok = rexi:stream2({meta, Meta}),
{ok, Acc};
view_cb({row, Row}, Acc) ->
+ %% TODO: distinguish between rows and docs
+ couch_cost:inc_doc(),
% Adding another row
ViewRow = #view_row{
id = couch_util:get_value(id, Row),
diff --git a/src/rexi/src/rexi.erl b/src/rexi/src/rexi.erl
index 77830996e..ffa5b2713 100644
--- a/src/rexi/src/rexi.erl
+++ b/src/rexi/src/rexi.erl
@@ -129,7 +129,8 @@ async_server_call(Server, Caller, Request) ->
-spec reply(any()) -> any().
reply(Reply) ->
{Caller, Ref} = get(rexi_from),
- erlang:send(Caller, {Ref, Reply}).
+ Cost = couch_cost:get_cost(),
+ erlang:send(Caller, {Ref,Reply,{cost,Cost}}).
%% @equiv sync_reply(Reply, 300000)
sync_reply(Reply) ->
@@ -214,7 +215,8 @@ stream(Msg, Limit, Timeout) ->
{ok, Count} ->
put(rexi_unacked, Count + 1),
{Caller, Ref} = get(rexi_from),
- erlang:send(Caller, {Ref, self(), Msg}),
+ Cost = couch_cost:get_cost(),
+ erlang:send(Caller, {Ref, self(), Msg, {cost, Cost}}),
ok
catch
throw:timeout ->
diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl
index d59c5ea0f..ae05bf7a6 100644
--- a/src/rexi/src/rexi_utils.erl
+++ b/src/rexi/src/rexi_utils.erl
@@ -83,7 +83,26 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
end;
{rexi, '$rexi_ping'} ->
{ok, Acc0};
+ {Ref, Msg, {cost,Cost}} ->
+ io:format("GOT COST: ~p -- ~p~n", [Cost, Msg]),
+ case lists:keyfind(Ref, Keypos, RefList) of
+ false ->
+ % this was some non-matching message which we will ignore
+ {ok, Acc0};
+ Worker ->
+ Fun(Msg, Worker, Acc0)
+ end;
+ {Ref, From, Msg, {cost,Cost}} ->
+ %%io:format("GOT COST: ~p~n", [Cost]),
+ io:format("GOT COST: ~p -- ~p~n", [Cost, Msg]),
+ case lists:keyfind(Ref, Keypos, RefList) of
+ false ->
+ {ok, Acc0};
+ Worker ->
+ Fun(Msg, {Worker, From}, Acc0)
+ end;
{Ref, Msg} ->
+ io:format("GOT NON COST MSG: ~p~n", [Msg]),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
% this was some non-matching message which we will ignore
@@ -92,6 +111,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) ->
Fun(Msg, Worker, Acc0)
end;
{Ref, From, Msg} ->
+ io:format("GOT NON COST MSG: ~p~n", [Msg]),
case lists:keyfind(Ref, Keypos, RefList) of
false ->
{ok, Acc0};