You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/12/05 15:00:04 UTC
[couchdb] branch prototype/fdb-layer-get-dbs-info updated (fec0dcb
-> 20cdcee)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard fec0dcb Track the size of data store in a database
discard f9ff19a Support `GET /_dbs_info` endpoint
new f8e1c2d Support `GET /_dbs_info` endpoint
new 20cdcee Track the size of data store in a database
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fec0dcb)
\
N -- N -- N refs/heads/prototype/fdb-layer-get-dbs-info (20cdcee)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd_misc.erl | 19 ++++++++-----------
1 file changed, 8 insertions(+), 11 deletions(-)
[couchdb] 01/02: Support `GET /_dbs_info` endpoint
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f8e1c2d97d3fc011e7f2096d924e0c9ab579ea0b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 13:44:35 2019 -0600
Support `GET /_dbs_info` endpoint
Previously only `POST` with a list of keys was supported. The new `GET`
support just dumps all database info blobs in a single ordered response.
---
src/chttpd/src/chttpd_misc.erl | 49 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 186ec9f..d7a2c46 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -157,6 +157,37 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
{ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
{ok, Acc#vacc{resp=Resp1}}.
+handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
+ ok = chttpd:verify_is_server_admin(Req),
+
+ #mrargs{
+ start_key = StartKey,
+ end_key = EndKey,
+ direction = Dir,
+ limit = Limit,
+ skip = Skip
+ } = couch_mrview_http:parse_params(Req, undefined),
+
+ Options = [
+ {start_key, StartKey},
+ {end_key, EndKey},
+ {dir, Dir},
+ {limit, Limit},
+ {skip, Skip}
+ ],
+
+ % TODO: Figure out if we can't calculate a valid
+ % ETag for this request. \xFFmetadataVersion won't
+ % work as we don't bump versions on size changes
+
+ {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
+ Callback = fun dbs_info_callback/2,
+ Acc = #vacc{req = Req, resp = Resp},
+ {ok, Resp} = fabric2_db:list_dbs_info(Callback, Acc, Options)
+ case is_record(Resp, vacc) of
+ true -> {ok, Resp#vacc.resp};
+ _ -> {ok, Resp}
+ end;
handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
chttpd:validate_ctype(Req, "application/json"),
Props = chttpd:json_body_obj(Req),
@@ -188,7 +219,23 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
send_chunk(Resp, "]"),
chttpd:end_json_response(Resp);
handle_dbs_info_req(Req) ->
- send_method_not_allowed(Req, "POST").
+ send_method_not_allowed(Req, "GET,HEAD,POST").
+
+dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
+ {ok, Acc#vacc{resp = Resp1}};
+dbs_info_callback({row, Props}, #vacc{resp = Resp0} = Acc) ->
+ Prepend = couch_mrview_http:prepend_val(Acc),
+ Chunk = [Prepend, ?JSON_ENCODE({Props})],
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+ {ok, Acc#vacc{prepend = ",", resp = Resp1}};
+dbs_info_callback(complete, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, Acc#vacc{resp = Resp2}};
+dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
+ {ok, Acc#vacc{resp = Resp1}}.
handle_task_status_req(#httpd{method='GET'}=Req) ->
ok = chttpd:verify_is_server_admin(Req),
[couchdb] 02/02: Track the size of data store in a database
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 20cdcee05fafcaa822e9961d5904f0add7fdbc62
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Dec 4 11:38:48 2019 -0600
Track the size of data store in a database
This tracks the number of bytes that would be required to store the
contents of a database as flat files on disk. Currently the following
items are tracked:
* Doc ids
* Revisions
* Doc body as JSON
* Attachment names
* Attachment type
* Attachment length
* Attachment md5s
* Attachment headers
* Local doc id
* Local doc revision
* Local doc bodies
---
src/couch/src/couch_att.erl | 19 ++++
src/fabric/include/fabric2.hrl | 8 +-
src/fabric/src/fabric2_db.erl | 6 +-
src/fabric/src/fabric2_fdb.erl | 157 ++++++++++++++++++++++++-----
src/fabric/test/fabric2_doc_crud_tests.erl | 5 +-
5 files changed, 163 insertions(+), 32 deletions(-)
diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index 2c33362..90d498c 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -27,6 +27,7 @@
]).
-export([
+ external_size/1,
size_info/1,
to_disk_term/1,
from_disk_term/3
@@ -179,6 +180,24 @@ merge_stubs([], _, Merged) ->
{ok, lists:reverse(Merged)}.
+external_size(Att) ->
+ NameSize = size(fetch(name, Att)),
+ TypeSize = case fetch(type, Att) of
+ undefined -> 0;
+ Type -> size(Type)
+ end,
+ AttSize = fetch(att_len, Att),
+ Md5Size = case fetch(md5, Att) of
+ undefined -> 0;
+ Md5 -> size(Md5)
+ end,
+ HeadersSize = case fetch(headers, Att) of
+ undefined -> 0;
+ Headers -> couch_ejson_size:encoded_size(Headers)
+ end,
+ NameSize + TypeSize + AttSize + Md5Size + HeadersSize.
+
+
size_info([]) ->
{ok, []};
size_info(Atts) ->
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 189995d..fc07f33 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -41,8 +41,14 @@
% Versions
--define(CURR_REV_FORMAT, 0).
+% 0 - Initial implementation
+% 1 - Added size information
+-define(CURR_REV_FORMAT, 1).
+
+% 0 - Adding local doc versions
+
+-define(CURR_LDOC_FORMAT, 0).
% Misc constants
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 688f794..1bc53ca 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1420,7 +1420,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
rev_id => {NewRevPos, NewRev},
rev_path => NewRevPath,
sequence => undefined,
- branch_count => undefined
+ branch_count => undefined,
+ rev_size => null
},
% Gather the list of possible winnig revisions
@@ -1475,7 +1476,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
rev_id => {RevPos, Rev},
rev_path => RevPath,
sequence => undefined,
- branch_count => undefined
+ branch_count => undefined,
+ rev_size => null
},
AllRevInfos = fabric2_fdb:get_all_revs(Db, DocId),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index c13ef7c..a91f82a 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -36,6 +36,7 @@
get_stat/2,
incr_stat/3,
+ incr_stat/4,
get_all_revs/2,
get_winning_revs/3,
@@ -451,6 +452,19 @@ incr_stat(#{} = Db, StatKey, Increment) when is_integer(Increment) ->
erlfdb:add(Tx, Key, Increment).
+incr_stat(_Db, _Section, _Key, 0) ->
+ ok;
+
+incr_stat(#{} = Db, Section, Key, Increment) when is_integer(Increment) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = ensure_current(Db),
+
+ BinKey = erlfdb_tuple:pack({?DB_STATS, Section, Key}, DbPrefix),
+ erlfdb:add(Tx, BinKey, Increment).
+
+
get_all_revs(#{} = Db, DocId) ->
#{
tx := Tx,
@@ -570,6 +584,15 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
get_local_doc_rev(_Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId, Val) ->
case Val of
+ <<255, RevBin/binary>> ->
+ % Versioned local docs
+ try
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev, _Size} -> Rev
+ end
+ catch _:_ ->
+ erlang:error({invalid_local_doc_rev, DocId, Val})
+ end;
<<131, _/binary>> ->
% Compatibility clause for an older encoding format
try binary_to_term(Val, [safe]) of
@@ -603,9 +626,16 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
deleted = Deleted
} = Doc,
+ % Doc body
+
+ {ok, RevSize} = write_doc_body(Db, Doc),
+
% Revision tree
- NewWinner = NewWinner0#{winner := true},
+ NewWinner = NewWinner0#{
+ winner := true,
+ rev_size := RevSize
+ },
NewRevId = maps:get(rev_id, NewWinner),
{WKey, WVal, WinnerVS} = revinfo_to_fdb(Tx, DbPrefix, DocId, NewWinner),
@@ -667,9 +697,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, NewRevId}),
erlfdb:set_versionstamped_key(Tx, NewSeqKey, NewSeqVal),
- % And all the rest...
-
- ok = write_doc_body(Db, Doc),
+ % Bump db version on design doc changes
IsDDoc = case Doc#doc.id of
<<?DESIGN_DOC_PREFIX, _/binary>> -> true;
@@ -680,6 +708,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
bump_db_version(Db)
end,
+ % Update our document counts
+
case UpdateStatus of
created ->
if not IsDDoc -> ok; true ->
@@ -706,6 +736,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ % Update database size
+ SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
+ Acc + case maps:get(rev_size, RI, null) of
+ null -> 0;
+ Size -> Size
+ end
+ end, 0, ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+
ok.
@@ -717,11 +756,18 @@ write_local_doc(#{} = Db0, Doc) ->
Id = Doc#doc.id,
- {LDocKey, LDocVal, Rows} = local_doc_to_fdb(Db, Doc),
+ {LDocKey, LDocVal, NewSize, Rows} = local_doc_to_fdb(Db, Doc),
- WasDeleted = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
- <<_/binary>> -> false;
- not_found -> true
+ {WasDeleted, PrevSize} = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
+ <<255, RevBin/binary>> ->
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, _Rev, Size} ->
+ {false, Size}
+ end;
+ <<_/binary>> ->
+ {false, 0};
+ not_found ->
+ {true, 0}
end,
BPrefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, Id}, DbPrefix),
@@ -747,6 +793,8 @@ write_local_doc(#{} = Db0, Doc) ->
ok
end,
+ incr_stat(Db, <<"sizes">>, <<"external">>, NewSize - PrevSize),
+
ok.
@@ -1010,9 +1058,11 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
+ {Rows, RevSize} = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, doc_to_fdb(Db, Doc)).
+ end, Rows),
+ {ok, RevSize}.
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1039,11 +1089,12 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
deleted := Deleted,
rev_id := {RevPos, Rev},
rev_path := RevPath,
- branch_count := BranchCount
+ branch_count := BranchCount,
+ rev_size := RevSize
} = RevId,
VS = new_versionstamp(Tx),
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
- Val = {?CURR_REV_FORMAT, VS, BranchCount, list_to_tuple(RevPath)},
+ Val = {?CURR_REV_FORMAT, VS, BranchCount, list_to_tuple(RevPath), RevSize},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack_vs(Val),
{KBin, VBin, VS};
@@ -1052,38 +1103,49 @@ revinfo_to_fdb(_Tx, DbPrefix, DocId, #{} = RevId) ->
#{
deleted := Deleted,
rev_id := {RevPos, Rev},
- rev_path := RevPath
+ rev_path := RevPath,
+ rev_size := RevSize
} = RevId,
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
- Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath)},
+ Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), RevSize},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack(Val),
{KBin, VBin, undefined}.
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, Sequence, BranchCount, RevPath} = Val,
+ {_RevFormat, Sequence, BranchCount, RevPath, RevSize} = Val,
#{
winner => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
sequence => Sequence,
- branch_count => BranchCount
+ branch_count => BranchCount,
+ rev_size => RevSize
};
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, RevPath} = Val,
+ {_RevFormat, RevPath, RevSize} = Val,
#{
winner => false,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
sequence => undefined,
- branch_count => undefined
- }.
+ branch_count => undefined,
+ rev_size => RevSize
+ };
+
+fdb_to_revinfo(Key, {0, S, B, R}) ->
+ Val = {?CURR_REV_FORMAT, S, B, R, null},
+ fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {0, R}) ->
+ Val = {?CURR_REV_FORMAT, R, null},
+ fdb_to_revinfo(Key, Val).
doc_to_fdb(Db, #doc{} = Doc) ->
@@ -1102,12 +1164,26 @@ doc_to_fdb(Db, #doc{} = Doc) ->
DiskAtts = lists:map(fun couch_att:to_disk_term/1, Atts),
Value = term_to_binary({Body, DiskAtts, Deleted}, [{minor_version, 1}]),
+ Chunks = chunkify_binary(Value),
{Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev, ChunkId}, DbPrefix),
{{Key, Chunk}, ChunkId + 1}
- end, 0, chunkify_binary(Value)),
- Rows.
+ end, 0, Chunks),
+
+ % Calculate the size of this revision
+ TotalSize = lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]),
+
+ {Rows, TotalSize}.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1157,8 +1233,28 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{{K, Chunk}, ChunkId + 1}
end, 0, chunkify_binary(BVal)),
- {Key, StoreRev, Rows}.
+ % Calculate size
+ TotalSize = case Doc#doc.deleted of
+ true ->
+ 0;
+ false ->
+ lists:sum([
+ size(Id),
+ size(StoreRev),
+ couch_ejson_size:encoded_size(Body)
+ ])
+ end,
+
+ RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
+ % Prefix our tuple encoding to make upgrades easier
+ Value = <<255, RawValue/binary>>,
+
+ {Key, Value, TotalSize, Rows}.
+
+
+fdb_to_local_doc(_Db, _DocId, not_found, []) ->
+ {not_found, missing};
fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
% This is an upgrade clause for the old encoding. We allow reading the old
@@ -1171,18 +1267,25 @@ fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
body = Body
};
-fdb_to_local_doc(_Db, _DocId, not_found, []) ->
- {not_found, missing};
+fdb_to_local_doc(_Db, DocId, <<255, RevBin/binary>>, Rows) when is_list(Rows) ->
+ Rev = case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev0, _Size} -> Rev0
+ end,
-fdb_to_local_doc(_Db, DocId, Rev, Rows) when is_list(Rows), is_binary(Rev) ->
BodyBin = iolist_to_binary(Rows),
Body = binary_to_term(BodyBin, [safe]),
+
#doc{
id = DocId,
revs = {0, [Rev]},
deleted = false,
body = Body
- }.
+ };
+
+fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
+ BaseRev = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, RawRev, 0}),
+ Rev = <<255, BaseRev/binary>>,
+ fdb_to_local_doc(Db, DocId, Rev, Rows).
chunkify_binary(Data) ->
diff --git a/src/fabric/test/fabric2_doc_crud_tests.erl b/src/fabric/test/fabric2_doc_crud_tests.erl
index 184eb4a..46cd4fc 100644
--- a/src/fabric/test/fabric2_doc_crud_tests.erl
+++ b/src/fabric/test/fabric2_doc_crud_tests.erl
@@ -884,11 +884,12 @@ local_doc_with_previous_encoding({Db, _}) ->
?assertEqual(NewBody, Doc3#doc.body),
% Old doc now has only the rev number in it
- OldDocBin = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ <<255, OldDocBin/binary>> = fabric2_fdb:transactional(Db, fun(TxDb) ->
#{tx := Tx} = TxDb,
erlfdb:wait(erlfdb:get(Tx, Key))
end),
- ?assertEqual(<<"2">> , OldDocBin).
+ Unpacked = erlfdb_tuple:unpack(OldDocBin),
+ ?assertMatch({?CURR_LDOC_FORMAT, <<"2">>, _}, Unpacked).
before_doc_update_skips_local_docs({Db0, _}) ->