You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/02/10 18:41:12 UTC
[couchdb] 14/16: fixup: add db size tracking
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit db2e8c4db21ab3d5a4f5af4b5db044833edd2f48
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:06 2020 -0600
fixup: add db size tracking
---
src/fabric/src/fabric2_db.erl | 7 +++--
src/fabric/src/fabric2_fdb.erl | 63 +++++++++++++++++++++++------------------
src/fabric/src/fabric2_util.erl | 21 ++++++++++++++
3 files changed, 61 insertions(+), 30 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 26aad75..4528194 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1417,13 +1417,14 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
NewRevInfo = #{
winner => undefined,
+ exists => false,
deleted => NewDeleted,
rev_id => {NewRevPos, NewRev},
rev_path => NewRevPath,
sequence => undefined,
branch_count => undefined,
att_hash => fabric2_util:hash_atts(Atts),
- rev_size => null
+ rev_size => fabric2_util:rev_size(Doc4)
},
% Gather the list of possible winnig revisions
@@ -1474,6 +1475,7 @@ update_doc_replicated(Db, Doc0, _Options) ->
DocRevInfo0 = #{
winner => undefined,
+ exists => false,
deleted => Deleted,
rev_id => {RevPos, Rev},
rev_path => RevPath,
@@ -1520,7 +1522,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
Doc2 = prep_and_validate(Db, Doc1, PrevRevInfo),
Doc3 = flush_doc_atts(Db, Doc2),
DocRevInfo2 = DocRevInfo1#{
- atts_hash => fabric2_util:hash_atts(Doc3#doc.atts)
+ atts_hash => fabric2_util:hash_atts(Doc3#doc.atts),
+ rev_size => fabric2_util:rev_size(Doc3)
},
% Possible winners are the previous winner and
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index f447e93..8d8616d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -632,7 +632,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Doc body
- {ok, RevSize} = write_doc_body(Db, Doc),
+ ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -663,8 +663,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Revision tree
NewWinner = NewWinner0#{
- winner := true,
- rev_size := RevSize
+ winner := true
},
NewRevId = maps:get(rev_id, NewWinner),
@@ -767,13 +766,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
end,
% Update database size
- SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
- Acc + case maps:get(rev_size, RI, null) of
- null -> 0;
- Size -> Size
- end
- end, 0, ToRemove),
- incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+ AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
+ RemSize = sum_rem_rev_sizes(ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, AddSize - RemSize),
ok.
@@ -1091,11 +1086,10 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
- {Rows, RevSize} = doc_to_fdb(Db, Doc),
+ Rows = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, Rows),
- {ok, RevSize}.
+ end, Rows).
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1208,6 +1202,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
{_RevFormat, Sequence, BranchCount, RevPath, AttHash, RevSize} = Val,
#{
winner => true,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1222,6 +1217,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
{_RevFormat, RevPath, AttHash, RevSize} = Val,
#{
winner => false,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1240,11 +1236,11 @@ fdb_to_revinfo(Key, {0, RPath}) ->
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, Seq, BCount, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val).
@@ -1271,19 +1267,7 @@ doc_to_fdb(Db, #doc{} = Doc) ->
{{Key, Chunk}, ChunkId + 1}
end, 0, Chunks),
- % Calculate the size of this revision
- TotalSize = lists:sum([
- size(Id),
- size(erlfdb_tuple:pack({Start})),
- size(Rev),
- 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
- couch_ejson_size:encoded_size(Body),
- lists:foldl(fun(Att, Acc) ->
- couch_att:external_size(Att) + Acc
- end, 0, Atts)
- ]),
-
- {Rows, TotalSize}.
+ Rows.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1388,6 +1372,29 @@ fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
fdb_to_local_doc(Db, DocId, Rev, Rows).
+sum_add_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := Exists,
+ rev_size := Size
+ } = RI,
+ case Exists of
+ true -> Acc;
+ false -> Size + Acc
+ end
+ end, 0, RevInfos).
+
+
+sum_rem_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := true,
+ rev_size := Size
+ } = RI,
+ Size + Acc
+ end, 0, RevInfos).
+
+
chunkify_binary(Data) ->
case Data of
<<>> ->
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 4e2e2d7..0f75390 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -17,6 +17,7 @@
revinfo_to_revs/1,
revinfo_to_path/1,
sort_revinfos/1,
+ rev_size/1,
seq_zero_vs/0,
seq_max_vs/0,
@@ -78,6 +79,26 @@ rev_sort_key(#{} = RevInfo) ->
{not Deleted, RevPos, Rev}.
+rev_size(#doc{} = Doc) ->
+ #doc{
+ id = Id,
+ revs = {Start, [Rev | _]},
+ body = Body,
+ atts = Atts
+ } = Doc,
+
+ lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]).
+
+
seq_zero_vs() ->
{versionstamp, 0, 0, 0}.