You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/02/12 23:17:16 UTC
[couchdb] branch prototype/fdb-layer-get-dbs-info updated (add2497
-> ae8d939)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard add2497 fixup: add size tests
discard 959ece7 fixup size tests
discard 2fc3059 fixup: add size tests
discard b728bce fixup: add size tracking
discard 1a7a33e fixup: db size tests
discard bf8f84b Fix doc attachment tests
discard db2e8c4 fixup: add db size tracking
discard 7d95d89 Add database size tests
discard 87e0795 Track the size of data stored in a database
discard 916a85f Support `GET /_dbs_info` endpoint
discard b2c6938 Implement `fabric2_db:list_dbs_info/1,2,3`
discard bde305d Implement async API for `fabric2_fdb:get_info/1`
discard c83b48c Track a database level view size rollup
discard 3565e52 Support jaeger http reporter
discard 24f5349 reserve search namespace
discard a7068fa Delete unused ets table creation
discard 5b1d506 Bump ioq to 2.1.3
discard f187627 Change map indexes to be stored in one row
discard 7507b68 Add `external` tag to opentrace events
discard 8918e28 Improve transaction name setting when tracing FDB transactions
add ab4eafa Improve transaction name setting when tracing FDB transactions
add 963f84b Add `external` tag to opentrace events
add 6b1da76 Merge pull request #2451 from cloudant/tracing-external
add ecaf215 Change map indexes to be stored in one row
add ea23fa1 Bump ioq to 2.1.3
add 34be5dd Merge pull request #2495 from apache/prototype/fdb-layer-bump-ioq
add b6c8a21 Delete unused ets table creation
add d81356f Merge pull request #2499 from apache/expiring-cache-cleanup
add 1496c49 reserve search namespace
add 7045d62 Merge pull request #2503 from apache/reserve-search-namespace
add bd3c021 Support jaeger http reporter
add 7e0c5bb Merge pull request #2494 from cloudant/add-http-reporter
add bf39736 Update httpotion to 3.1.3
add 8650a4e Support setting base_url in Couch test helper
add 3b9f04d fix b3 - Headers suppose to be strings
add d233f81 Add basic test case for b3 fix
add f431566 Merge pull request #2519 from cloudant/fix-b3-header
new b2b5208 Track a database level view size rollup
new b8b13ec Implement async API for `fabric2_fdb:get_info/1`
new 7505bfd Implement `fabric2_db:list_dbs_info/1,2,3`
new 3d4970f Support `GET /_dbs_info` endpoint
new 9cecf8a Track the size of data stored in a database
new 04cbdbd Add database size tests
new a50fbfa fixup: add db size tracking
new 1cfd2a5 Fix doc attachment tests
new 052610c fixup: db size tests
new 0161223 fixup: add size tracking
new 9d5062b fixup: add size tests
new a472206 fixup size tests
new ae8d939 fixup: add size tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (add2497)
\
N -- N -- N refs/heads/prototype/fdb-layer-get-dbs-info (ae8d939)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
mix.exs | 2 +-
mix.lock | 2 +-
src/chttpd/src/chttpd.erl | 2 +-
src/{couch => chttpd}/test/exunit/test_helper.exs | 0
src/chttpd/test/exunit/tracing_test.exs | 101 ++++++++++++++++++
test/elixir/lib/couch.ex | 124 +++-------------------
6 files changed, 119 insertions(+), 112 deletions(-)
copy src/{couch => chttpd}/test/exunit/test_helper.exs (100%)
create mode 100644 src/chttpd/test/exunit/tracing_test.exs
[couchdb] 10/13: fixup: add size tracking
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0161223c6acb3204c520044efdd2c509b2596809
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 16:41:33 2020 -0600
fixup: add size tracking
---
src/fabric/src/fabric2_fdb.erl | 17 +++--------------
src/fabric/src/fabric2_util.erl | 33 ++++++++++++++++++++++++++++++++-
2 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 8d8616d..d5be6d7 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -1317,24 +1317,13 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{{K, Chunk}, ChunkId + 1}
end, 0, chunkify_binary(BVal)),
- % Calculate size
- TotalSize = case Doc#doc.deleted of
- true ->
- 0;
- false ->
- lists:sum([
- size(Id),
- size(StoreRev),
- couch_ejson_size:encoded_size(Body)
- ])
- end,
-
- RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
+ NewSize = fabric2_util:ldoc_size(Doc),
+ RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, NewSize}),
% Prefix our tuple encoding to make upgrades easier
Value = <<255, RawValue/binary>>,
- {Key, Value, TotalSize, Rows}.
+ {Key, Value, NewSize, Rows}.
fdb_to_local_doc(_Db, _DocId, not_found, []) ->
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 0f75390..766441c 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -18,6 +18,7 @@
revinfo_to_path/1,
sort_revinfos/1,
rev_size/1,
+ ldoc_size/1,
seq_zero_vs/0,
seq_max_vs/0,
@@ -82,11 +83,16 @@ rev_sort_key(#{} = RevInfo) ->
rev_size(#doc{} = Doc) ->
#doc{
id = Id,
- revs = {Start, [Rev | _]},
+ revs = Revs,
body = Body,
atts = Atts
} = Doc,
+ {Start, Rev} = case Revs of
+ {0, []} -> {0, <<>>};
+ {N, [RevId | _]} -> {N, RevId}
+ end,
+
lists:sum([
size(Id),
size(erlfdb_tuple:pack({Start})),
@@ -99,6 +105,31 @@ rev_size(#doc{} = Doc) ->
]).
+ldoc_size(#doc{id = <<"_local/">>} = Doc) ->
+ #doc{
+ id = Id,
+ revs = {0, [Rev]},
+ deleted = Deleted,
+ body = Body
+ } = Doc,
+
+ StoreRev = case Rev of
+ _ when is_integer(Rev) -> integer_to_binary(Rev);
+ _ when is_binary(Rev) -> Rev
+ end,
+
+ case Deleted of
+ true ->
+ 0;
+ false ->
+ lists:sum([
+ size(Id),
+ size(StoreRev),
+ couch_ejson_size:encoded_size(Body)
+ ])
+ end.
+
+
seq_zero_vs() ->
{versionstamp, 0, 0, 0}.
[couchdb] 12/13: fixup size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a472206dfd940cb1c4f40d61384918fe216c8eb7
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Feb 12 15:01:19 2020 -0600
fixup size tests
---
src/fabric/src/fabric2_util.erl | 2 +-
src/fabric/test/fabric2_doc_size_tests.erl | 78 +++++++++++++++++++++++++++---
2 files changed, 73 insertions(+), 7 deletions(-)
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 766441c..f51c6e8 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -105,7 +105,7 @@ rev_size(#doc{} = Doc) ->
]).
-ldoc_size(#doc{id = <<"_local/">>} = Doc) ->
+ldoc_size(#doc{id = <<"_local/", _/binary>>} = Doc) ->
#doc{
id = Id,
revs = {0, [Rev]},
diff --git a/src/fabric/test/fabric2_doc_size_tests.erl b/src/fabric/test/fabric2_doc_size_tests.erl
index 37f1740..96ecf74 100644
--- a/src/fabric/test/fabric2_doc_size_tests.erl
+++ b/src/fabric/test/fabric2_doc_size_tests.erl
@@ -106,7 +106,6 @@
>>}
]).
-
-define(ATT_HEADERS, [
{0, undefined},
{2, {[]}},
@@ -114,6 +113,24 @@
{32, {[{<<"a">>, <<"header">>}, {<<"b">>, <<"such header">>}]}}
]).
+-define(LDOC_IDS, [
+ {7, <<"_local/">>},
+ {8, <<"_local/a">>},
+ {10, <<"_local/foo">>},
+ {13, <<"_local/foobar">>},
+ {39, <<"_local/af196ae095631b020eedf8f69303e336">>}
+]).
+
+-define(LDOC_REVS, [
+ {1, <<"0">>},
+ {2, <<"10">>},
+ {3, <<"100">>},
+ {4, <<"1000">>},
+ {5, <<"10000">>},
+ {6, <<"100000">>},
+ {7, <<"1000000">>}
+]).
+
empty_doc_test() ->
?assertEqual(4, fabric2_util:rev_size(#doc{})).
@@ -184,7 +201,27 @@ att_headers_test() ->
end, ?ATT_HEADERS).
-combinatorics_test() ->
+local_doc_ids_test() ->
+ lists:foreach(fun({Size, LDocId}) ->
+ ?assertEqual(3 + Size, fabric2_util:ldoc_size(mk_ldoc(LDocId, 0)))
+ end, ?LDOC_IDS).
+
+
+local_doc_revs_test() ->
+ lists:foreach(fun({Size, Rev}) ->
+ Doc = mk_ldoc(<<"_local/foo">>, Rev),
+ ?assertEqual(12 + Size, fabric2_util:ldoc_size(Doc))
+ end, ?LDOC_REVS).
+
+
+local_doc_bodies_test() ->
+ lists:foreach(fun({Size, Body}) ->
+ Doc = mk_ldoc(<<"_local/foo">>, 0, Body),
+ ?assertEqual(11 + Size, fabric2_util:ldoc_size(Doc))
+ end, ?BODIES).
+
+
+doc_combinatorics_test() ->
Elements = [
{?DOC_IDS, fun(Doc, DocId) -> Doc#doc{id = DocId} end},
{?REV_STARTS, fun(Doc, RevStart) ->
@@ -198,18 +235,35 @@ combinatorics_test() ->
{?DELETED, fun(Doc, Deleted) -> Doc#doc{deleted = Deleted} end},
{?BODIES, fun(Doc, Body) -> Doc#doc{body = Body} end}
],
- combine(Elements, 0, #doc{}).
+ doc_combine(Elements, 0, #doc{}).
-combine([], TotalSize, Doc) ->
+doc_combine([], TotalSize, Doc) ->
?assertEqual(TotalSize, fabric2_util:rev_size(Doc));
-combine([{Elems, UpdateFun} | Rest], TotalSize, Doc) ->
+doc_combine([{Elems, UpdateFun} | Rest], TotalSize, Doc) ->
lists:foreach(fun({Size, Elem}) ->
- combine(Rest, TotalSize + Size, UpdateFun(Doc, Elem))
+ doc_combine(Rest, TotalSize + Size, UpdateFun(Doc, Elem))
end, Elems).
+local_doc_combinatorics_test() ->
+ Elements = [
+ {?LDOC_IDS, fun(Doc, DocId) -> Doc#doc{id = DocId} end},
+ {?LDOC_REVS, fun(Doc, Rev) -> Doc#doc{revs = {0, [Rev]}} end},
+ {?BODIES, fun(Doc, Body) -> Doc#doc{body = Body} end}
+ ],
+ local_doc_combine(Elements, 0, #doc{}).
+
+
+local_doc_combine([], TotalSize, Doc) ->
+ ?assertEqual(TotalSize, fabric2_util:ldoc_size(Doc));
+
+local_doc_combine([{Elems, UpdateFun} | Rest], TotalSize, Doc) ->
+ lists:foreach(fun({Size, Elem}) ->
+ local_doc_combine(Rest, TotalSize + Size, UpdateFun(Doc, Elem))
+ end, Elems).
+
random_docs_test() ->
lists:foreach(fun(_) ->
{DocIdSize, DocId} = choose(?DOC_IDS),
@@ -269,6 +323,18 @@ mk_att(Name, Type, Data, AddMd5, Headers) ->
]).
+mk_ldoc(DocId, Rev) ->
+ mk_ldoc(DocId, Rev, {[]}).
+
+
+mk_ldoc(DocId, Rev, Body) ->
+ #doc{
+ id = DocId,
+ revs = {0, [Rev]},
+ body = Body
+ }.
+
+
choose(Options) ->
Pos = rand:uniform(length(Options)),
lists:nth(Pos, Options).
[couchdb] 05/13: Track the size of data stored in a database
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9cecf8a664627b953b65311cb48618af632429ae
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Dec 4 11:38:48 2019 -0600
Track the size of data stored in a database
This tracks the number of bytes that would be required to store the
contents of a database as flat files on disk. Currently the following
items are tracked:
* Doc ids
* Revisions
* Doc body as JSON
* Attachment names
* Attachment type
* Attachment length
* Attachment md5s
* Attachment headers
* Local doc id
* Local doc revision
* Local doc bodies
---
src/couch/src/couch_att.erl | 19 ++++
src/fabric/include/fabric2.hrl | 7 +-
src/fabric/src/fabric2_db.erl | 6 +-
src/fabric/src/fabric2_fdb.erl | 152 ++++++++++++++++++++++++-----
src/fabric/test/fabric2_doc_crud_tests.erl | 5 +-
5 files changed, 159 insertions(+), 30 deletions(-)
diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index 2c33362..90d498c 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -27,6 +27,7 @@
]).
-export([
+ external_size/1,
size_info/1,
to_disk_term/1,
from_disk_term/3
@@ -179,6 +180,24 @@ merge_stubs([], _, Merged) ->
{ok, lists:reverse(Merged)}.
+external_size(Att) ->
+ NameSize = size(fetch(name, Att)),
+ TypeSize = case fetch(type, Att) of
+ undefined -> 0;
+ Type -> size(Type)
+ end,
+ AttSize = fetch(att_len, Att),
+ Md5Size = case fetch(md5, Att) of
+ undefined -> 0;
+ Md5 -> size(Md5)
+ end,
+ HeadersSize = case fetch(headers, Att) of
+ undefined -> 0;
+ Headers -> couch_ejson_size:encoded_size(Headers)
+ end,
+ NameSize + TypeSize + AttSize + Md5Size + HeadersSize.
+
+
size_info([]) ->
{ok, []};
size_info(Atts) ->
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 828a51b..5f2571e 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -45,8 +45,13 @@
% 0 - Initial implementation
% 1 - Added attachment hash
+% 2 - Added size information
--define(CURR_REV_FORMAT, 1).
+-define(CURR_REV_FORMAT, 2).
+
+% 0 - Adding local doc versions
+
+-define(CURR_LDOC_FORMAT, 0).
% Misc constants
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 17c899d..26aad75 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1422,7 +1422,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
rev_path => NewRevPath,
sequence => undefined,
branch_count => undefined,
- att_hash => fabric2_util:hash_atts(Atts)
+ att_hash => fabric2_util:hash_atts(Atts),
+ rev_size => null
},
% Gather the list of possible winnig revisions
@@ -1478,7 +1479,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
rev_path => RevPath,
sequence => undefined,
branch_count => undefined,
- att_hash => <<>>
+ att_hash => <<>>,
+ rev_size => null
},
AllRevInfos = fabric2_fdb:get_all_revs(Db, DocId),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 99611b0..f447e93 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -36,6 +36,7 @@
get_stat/2,
incr_stat/3,
+ incr_stat/4,
get_all_revs/2,
get_winning_revs/3,
@@ -454,6 +455,19 @@ incr_stat(#{} = Db, StatKey, Increment) when is_integer(Increment) ->
erlfdb:add(Tx, Key, Increment).
+incr_stat(_Db, _Section, _Key, 0) ->
+ ok;
+
+incr_stat(#{} = Db, Section, Key, Increment) when is_integer(Increment) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = ensure_current(Db),
+
+ BinKey = erlfdb_tuple:pack({?DB_STATS, Section, Key}, DbPrefix),
+ erlfdb:add(Tx, BinKey, Increment).
+
+
get_all_revs(#{} = Db, DocId) ->
#{
tx := Tx,
@@ -573,6 +587,15 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
get_local_doc_rev(_Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId, Val) ->
case Val of
+ <<255, RevBin/binary>> ->
+ % Versioned local docs
+ try
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev, _Size} -> Rev
+ end
+ catch _:_ ->
+ erlang:error({invalid_local_doc_rev, DocId, Val})
+ end;
<<131, _/binary>> ->
% Compatibility clause for an older encoding format
try binary_to_term(Val, [safe]) of
@@ -609,7 +632,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Doc body
- ok = write_doc_body(Db, Doc),
+ {ok, RevSize} = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -639,7 +662,10 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Revision tree
- NewWinner = NewWinner0#{winner := true},
+ NewWinner = NewWinner0#{
+ winner := true,
+ rev_size := RevSize
+ },
NewRevId = maps:get(rev_id, NewWinner),
{WKey, WVal, WinnerVS} = revinfo_to_fdb(Tx, DbPrefix, DocId, NewWinner),
@@ -701,7 +727,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, NewRevId}),
erlfdb:set_versionstamped_key(Tx, NewSeqKey, NewSeqVal),
- % And all the rest...
+ % Bump db version on design doc changes
IsDDoc = case Doc#doc.id of
<<?DESIGN_DOC_PREFIX, _/binary>> -> true;
@@ -712,6 +738,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
bump_db_version(Db)
end,
+ % Update our document counts
+
case UpdateStatus of
created ->
if not IsDDoc -> ok; true ->
@@ -738,6 +766,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ % Update database size
+ SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
+ Acc + case maps:get(rev_size, RI, null) of
+ null -> 0;
+ Size -> Size
+ end
+ end, 0, ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+
ok.
@@ -749,11 +786,18 @@ write_local_doc(#{} = Db0, Doc) ->
Id = Doc#doc.id,
- {LDocKey, LDocVal, Rows} = local_doc_to_fdb(Db, Doc),
+ {LDocKey, LDocVal, NewSize, Rows} = local_doc_to_fdb(Db, Doc),
- WasDeleted = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
- <<_/binary>> -> false;
- not_found -> true
+ {WasDeleted, PrevSize} = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
+ <<255, RevBin/binary>> ->
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, _Rev, Size} ->
+ {false, Size}
+ end;
+ <<_/binary>> ->
+ {false, 0};
+ not_found ->
+ {true, 0}
end,
BPrefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, Id}, DbPrefix),
@@ -779,6 +823,8 @@ write_local_doc(#{} = Db0, Doc) ->
ok
end,
+ incr_stat(Db, <<"sizes">>, <<"external">>, NewSize - PrevSize),
+
ok.
@@ -1045,9 +1091,11 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
+ {Rows, RevSize} = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, doc_to_fdb(Db, Doc)).
+ end, Rows),
+ {ok, RevSize}.
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1123,7 +1171,8 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
rev_id := {RevPos, Rev},
rev_path := RevPath,
branch_count := BranchCount,
- att_hash := AttHash
+ att_hash := AttHash,
+ rev_size := RevSize
} = RevId,
VS = new_versionstamp(Tx),
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
@@ -1132,7 +1181,8 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
VS,
BranchCount,
list_to_tuple(RevPath),
- AttHash
+ AttHash,
+ RevSize
},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack_vs(Val),
@@ -1143,18 +1193,19 @@ revinfo_to_fdb(_Tx, DbPrefix, DocId, #{} = RevId) ->
deleted := Deleted,
rev_id := {RevPos, Rev},
rev_path := RevPath,
- att_hash := AttHash
+ att_hash := AttHash,
+ rev_size := RevSize
} = RevId,
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
- Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), AttHash},
+ Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), AttHash, RevSize},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack(Val),
{KBin, VBin, undefined}.
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, Sequence, BranchCount, RevPath, AttHash} = Val,
+ {_RevFormat, Sequence, BranchCount, RevPath, AttHash, RevSize} = Val,
#{
winner => true,
deleted => not NotDeleted,
@@ -1162,12 +1213,13 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
rev_path => tuple_to_list(RevPath),
sequence => Sequence,
branch_count => BranchCount,
- att_hash => AttHash
+ att_hash => AttHash,
+ rev_size => RevSize
};
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, RevPath, AttHash} = Val,
+ {_RevFormat, RevPath, AttHash, RevSize} = Val,
#{
winner => false,
deleted => not NotDeleted,
@@ -1175,7 +1227,8 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val) ->
rev_path => tuple_to_list(RevPath),
sequence => undefined,
branch_count => undefined,
- att_hash => AttHash
+ att_hash => AttHash,
+ rev_size => RevSize
};
fdb_to_revinfo(Key, {0, Seq, BCount, RPath}) ->
@@ -1184,6 +1237,14 @@ fdb_to_revinfo(Key, {0, Seq, BCount, RPath}) ->
fdb_to_revinfo(Key, {0, RPath}) ->
Val = {?CURR_REV_FORMAT, RPath, <<>>},
+ fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {1, Seq, BCount, RPath, AttHash}) ->
+ Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, null},
+ fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
+ Val = {?CURR_REV_FORMAT, RPath, AttHash, null},
fdb_to_revinfo(Key, Val).
@@ -1203,12 +1264,26 @@ doc_to_fdb(Db, #doc{} = Doc) ->
DiskAtts = lists:map(fun couch_att:to_disk_term/1, Atts),
Value = term_to_binary({Body, DiskAtts, Deleted}, [{minor_version, 1}]),
+ Chunks = chunkify_binary(Value),
{Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev, ChunkId}, DbPrefix),
{{Key, Chunk}, ChunkId + 1}
- end, 0, chunkify_binary(Value)),
- Rows.
+ end, 0, Chunks),
+
+ % Calculate the size of this revision
+ TotalSize = lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]),
+
+ {Rows, TotalSize}.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1258,9 +1333,29 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{{K, Chunk}, ChunkId + 1}
end, 0, chunkify_binary(BVal)),
- {Key, StoreRev, Rows}.
+ % Calculate size
+ TotalSize = case Doc#doc.deleted of
+ true ->
+ 0;
+ false ->
+ lists:sum([
+ size(Id),
+ size(StoreRev),
+ couch_ejson_size:encoded_size(Body)
+ ])
+ end,
+
+ RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
+
+ % Prefix our tuple encoding to make upgrades easier
+ Value = <<255, RawValue/binary>>,
+
+ {Key, Value, TotalSize, Rows}.
+fdb_to_local_doc(_Db, _DocId, not_found, []) ->
+ {not_found, missing};
+
fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
% This is an upgrade clause for the old encoding. We allow reading the old
% value and will perform an upgrade of the storage format on an update.
@@ -1272,18 +1367,25 @@ fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
body = Body
};
-fdb_to_local_doc(_Db, _DocId, not_found, []) ->
- {not_found, missing};
+fdb_to_local_doc(_Db, DocId, <<255, RevBin/binary>>, Rows) when is_list(Rows) ->
+ Rev = case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev0, _Size} -> Rev0
+ end,
-fdb_to_local_doc(_Db, DocId, Rev, Rows) when is_list(Rows), is_binary(Rev) ->
BodyBin = iolist_to_binary(Rows),
Body = binary_to_term(BodyBin, [safe]),
+
#doc{
id = DocId,
revs = {0, [Rev]},
deleted = false,
body = Body
- }.
+ };
+
+fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
+ BaseRev = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, RawRev, 0}),
+ Rev = <<255, BaseRev/binary>>,
+ fdb_to_local_doc(Db, DocId, Rev, Rows).
chunkify_binary(Data) ->
diff --git a/src/fabric/test/fabric2_doc_crud_tests.erl b/src/fabric/test/fabric2_doc_crud_tests.erl
index 184eb4a..46cd4fc 100644
--- a/src/fabric/test/fabric2_doc_crud_tests.erl
+++ b/src/fabric/test/fabric2_doc_crud_tests.erl
@@ -884,11 +884,12 @@ local_doc_with_previous_encoding({Db, _}) ->
?assertEqual(NewBody, Doc3#doc.body),
% Old doc now has only the rev number in it
- OldDocBin = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ <<255, OldDocBin/binary>> = fabric2_fdb:transactional(Db, fun(TxDb) ->
#{tx := Tx} = TxDb,
erlfdb:wait(erlfdb:get(Tx, Key))
end),
- ?assertEqual(<<"2">> , OldDocBin).
+ Unpacked = erlfdb_tuple:unpack(OldDocBin),
+ ?assertMatch({?CURR_LDOC_FORMAT, <<"2">>, _}, Unpacked).
before_doc_update_skips_local_docs({Db0, _}) ->
[couchdb] 03/13: Implement `fabric2_db:list_dbs_info/1,2,3`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7505bfd76f8bd4e373bfeb01f045f7d298690298
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:45:36 2019 -0600
Implement `fabric2_db:list_dbs_info/1,2,3`
This API allows for listing all database info blobs in a single request.
It accepts the same parameters as `_all_dbs` for controlling pagination
of results and so on.
---
src/fabric/src/fabric2_db.erl | 100 +++++++++++++++++++++++++-----
src/fabric/src/fabric2_fdb.erl | 11 ++++
src/fabric/test/fabric2_db_crud_tests.erl | 31 ++++++++-
3 files changed, 126 insertions(+), 16 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 6d015df..17c899d 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -22,6 +22,10 @@
list_dbs/1,
list_dbs/3,
+ list_dbs_info/0,
+ list_dbs_info/1,
+ list_dbs_info/3,
+
check_is_admin/1,
check_is_member/1,
@@ -238,6 +242,46 @@ list_dbs(UserFun, UserAcc0, Options) ->
end).
+list_dbs_info() ->
+ list_dbs_info([]).
+
+
+list_dbs_info(Options) ->
+ Callback = fun(Value, Acc) ->
+ NewAcc = case Value of
+ {meta, _} -> Acc;
+ {row, DbInfo} -> [DbInfo | Acc];
+ complete -> Acc
+ end,
+ {ok, NewAcc}
+ end,
+ {ok, DbInfos} = list_dbs_info(Callback, [], Options),
+ {ok, lists:reverse(DbInfos)}.
+
+
+list_dbs_info(UserFun, UserAcc0, Options) ->
+ FoldFun = fun(DbName, InfoFuture, {FutureQ, Count, Acc}) ->
+ NewFutureQ = queue:in({DbName, InfoFuture}, FutureQ),
+ drain_info_futures(NewFutureQ, Count + 1, UserFun, Acc)
+ end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ try
+ UserAcc1 = maybe_stop(UserFun({meta, []}, UserAcc0)),
+ InitAcc = {queue:new(), 0, UserAcc1},
+ {FinalFutureQ, _, UserAcc2} = fabric2_fdb:list_dbs_info(
+ Tx,
+ FoldFun,
+ InitAcc,
+ Options
+ ),
+ UserAcc3 = drain_all_info_futures(FinalFutureQ, UserFun, UserAcc2),
+ {ok, maybe_stop(UserFun(complete, UserAcc3))}
+ catch throw:{stop, FinalUserAcc} ->
+ {ok, FinalUserAcc}
+ end
+ end).
+
+
is_admin(Db, {SecProps}) when is_list(SecProps) ->
case fabric2_db_plugin:check_is_admin(Db) of
true ->
@@ -313,21 +357,7 @@ get_db_info(#{} = Db) ->
DbProps = fabric2_fdb:transactional(Db, fun(TxDb) ->
fabric2_fdb:get_info(TxDb)
end),
-
- BaseProps = [
- {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
- {compact_running, false},
- {data_size, 0},
- {db_name, name(Db)},
- {disk_format_version, 0},
- {disk_size, 0},
- {instance_start_time, <<"0">>},
- {purge_seq, 0}
- ],
-
- {ok, lists:foldl(fun({Key, Val}, Acc) ->
- lists:keystore(Key, 1, Acc, {Key, Val})
- end, BaseProps, DbProps)}.
+ {ok, make_db_info(name(Db), DbProps)}.
get_del_doc_count(#{} = Db) ->
@@ -944,6 +974,46 @@ maybe_add_sys_db_callbacks(Db) ->
}.
+make_db_info(DbName, Props) ->
+ BaseProps = [
+ {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
+ {compact_running, false},
+ {data_size, 0},
+ {db_name, DbName},
+ {disk_format_version, 0},
+ {disk_size, 0},
+ {instance_start_time, <<"0">>},
+ {purge_seq, 0}
+ ],
+
+ lists:foldl(fun({Key, Val}, Acc) ->
+ lists:keystore(Key, 1, Acc, {Key, Val})
+ end, BaseProps, Props).
+
+
+drain_info_futures(FutureQ, Count, _UserFun, Acc) when Count < 100 ->
+ {FutureQ, Count, Acc};
+
+drain_info_futures(FutureQ, Count, UserFun, Acc) when Count >= 100 ->
+ {{value, {DbName, Future}}, RestQ} = queue:out(FutureQ),
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ {RestQ, Count - 1, NewAcc}.
+
+
+drain_all_info_futures(FutureQ, UserFun, Acc) ->
+ case queue:out(FutureQ) of
+ {{value, {DbName, Future}}, RestQ} ->
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ drain_all_info_futures(RestQ, UserFun, NewAcc);
+ {empty, _} ->
+ Acc
+ end.
+
+
new_revid(Db, Doc) ->
#doc{
id = DocId,
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 0e7cba8..99611b0 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -27,6 +27,7 @@
get_dir/1,
list_dbs/4,
+ list_dbs_info/4,
get_info/1,
get_info_future/2,
@@ -330,6 +331,16 @@ list_dbs(Tx, Callback, AccIn, Options) ->
end, AccIn, Options).
+list_dbs_info(Tx, Callback, AccIn, Options) ->
+ LayerPrefix = get_dir(Tx),
+ Prefix = erlfdb_tuple:pack({?ALL_DBS}, LayerPrefix),
+ fold_range({tx, Tx}, Prefix, fun({DbNameKey, DbPrefix}, Acc) ->
+ {DbName} = erlfdb_tuple:unpack(DbNameKey, Prefix),
+ InfoFuture = get_info_future(Tx, DbPrefix),
+ Callback(DbName, InfoFuture, Acc)
+ end, AccIn, Options).
+
+
get_info(#{} = Db) ->
#{
tx := Tx,
diff --git a/src/fabric/test/fabric2_db_crud_tests.erl b/src/fabric/test/fabric2_db_crud_tests.erl
index cc44f7d..8052551 100644
--- a/src/fabric/test/fabric2_db_crud_tests.erl
+++ b/src/fabric/test/fabric2_db_crud_tests.erl
@@ -29,7 +29,8 @@ crud_test_() ->
?TDEF(create_db),
?TDEF(open_db),
?TDEF(delete_db),
- ?TDEF(list_dbs)
+ ?TDEF(list_dbs),
+ ?TDEF(list_dbs_info)
])
}
}.
@@ -84,3 +85,31 @@ list_dbs(_) ->
?assertEqual(ok, fabric2_db:delete(DbName, [])),
AllDbs3 = fabric2_db:list_dbs(),
?assert(not lists:member(DbName, AllDbs3)).
+
+
+list_dbs_info(_) ->
+ DbName = ?tempdb(),
+ {ok, AllDbInfos1} = fabric2_db:list_dbs_info(),
+
+ ?assert(is_list(AllDbInfos1)),
+ ?assert(not is_db_info_member(DbName, AllDbInfos1)),
+
+ ?assertMatch({ok, _}, fabric2_db:create(DbName, [])),
+ {ok, AllDbInfos2} = fabric2_db:list_dbs_info(),
+ ?assert(is_db_info_member(DbName, AllDbInfos2)),
+
+ ?assertEqual(ok, fabric2_db:delete(DbName, [])),
+ {ok, AllDbInfos3} = fabric2_db:list_dbs_info(),
+ ?assert(not is_db_info_member(DbName, AllDbInfos3)).
+
+
+is_db_info_member(_, []) ->
+ false;
+
+is_db_info_member(DbName, [DbInfo | RestInfos]) ->
+ case lists:keyfind(db_name, 1, DbInfo) of
+ {db_name, DbName} ->
+ true;
+ _E ->
+ is_db_info_member(DbName, RestInfos)
+ end.
[couchdb] 11/13: fixup: add size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9d5062bf8f9b9b92db54e7b28d12d072ad560a99
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 16:41:42 2020 -0600
fixup: add size tests
---
src/fabric/test/fabric2_doc_size_tests.erl | 274 +++++++++++++++++++++++++++++
1 file changed, 274 insertions(+)
diff --git a/src/fabric/test/fabric2_doc_size_tests.erl b/src/fabric/test/fabric2_doc_size_tests.erl
new file mode 100644
index 0000000..37f1740
--- /dev/null
+++ b/src/fabric/test/fabric2_doc_size_tests.erl
@@ -0,0 +1,274 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_doc_size_tests).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+% Doc body size calculations
+% ID: size(Doc#doc.id)
+% Rev: size(erlfdb_tuple:encode(Start)) + size(Rev) % where Rev is usually 16
+% Deleted: 1 % (binary value is one byte)
+% Body: couch_ejson_size:external_size(Body) % Where empty is {} which is 2)
+
+
+-define(NUM_RANDOM_TESTS, 1000).
+
+
+-define(DOC_IDS, [
+ {0, <<>>},
+ {1, <<"a">>},
+ {3, <<"foo">>},
+ {6, <<"foobar">>},
+ {32, <<"af196ae095631b020eedf8f69303e336">>}
+]).
+
+-define(REV_STARTS, [
+ {1, 0},
+ {2, 1},
+ {2, 255},
+ {3, 256},
+ {3, 65535},
+ {4, 65536},
+ {4, 16777215},
+ {5, 16777216},
+ {5, 4294967295},
+ {6, 4294967296},
+ {6, 1099511627775},
+ {7, 1099511627776},
+ {7, 281474976710655},
+ {8, 281474976710656},
+ {8, 72057594037927935},
+ {9, 72057594037927936},
+ {9, 18446744073709551615},
+
+ % The jump from 9 to 11 bytes is because when we
+ % spill over into the bigint range of 9-255
+ % bytes we have an extra byte that encodes the
+ % length of the bigint.
+ {11, 18446744073709551616}
+]).
+
+-define(REVS, [
+ {0, <<>>},
+ {8, <<"foobarba">>},
+ {16, <<"foobarbazbambang">>}
+]).
+
+-define(DELETED, [
+ {1, true},
+ {1, false}
+]).
+
+-define(BODIES, [
+ {2, {[]}},
+ {13, {[{<<"foo">>, <<"bar">>}]}},
+ {28, {[{<<"b">>, <<"a">>}, {<<"c">>, [true, null, []]}]}}
+]).
+
+-define(ATT_NAMES, [
+ {5, <<"a.txt">>},
+ {7, <<"foo.csv">>},
+ {29, <<"a-longer-name-for-example.bat">>}
+]).
+
+-define(ATT_TYPES, [
+ {24, <<"application/octet-stream">>},
+ {10, <<"text/plain">>},
+ {9, <<"image/png">>}
+]).
+
+-define(ATT_BODIES, [
+ {0, <<>>},
+ {1, <<"g">>},
+ {6, <<"foobar">>},
+ {384, <<
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ "xlasdjfsapoiewrposdlfadfuaducvwerwlkdsfljdfusfsd"
+ >>}
+]).
+
+
+-define(ATT_HEADERS, [
+ {0, undefined},
+ {2, {[]}},
+ {13, {[{<<"foo">>, <<"bar">>}]}},
+ {32, {[{<<"a">>, <<"header">>}, {<<"b">>, <<"such header">>}]}}
+]).
+
+
+empty_doc_test() ->
+ ?assertEqual(4, fabric2_util:rev_size(#doc{})).
+
+
+docid_size_test() ->
+ lists:foreach(fun({Size, DocId}) ->
+ ?assertEqual(4 + Size, fabric2_util:rev_size(#doc{id = DocId}))
+ end, ?DOC_IDS).
+
+
+rev_size_test() ->
+ lists:foreach(fun({StartSize, Start}) ->
+ lists:foreach(fun({RevSize, Rev}) ->
+ Doc = #doc{
+ revs = {Start, [Rev]}
+ },
+ ?assertEqual(3 + StartSize + RevSize, fabric2_util:rev_size(Doc))
+ end, ?REVS)
+ end, ?REV_STARTS).
+
+
+deleted_size_test() ->
+ lists:foreach(fun({Size, Deleted}) ->
+ ?assertEqual(3 + Size, fabric2_util:rev_size(#doc{deleted = Deleted}))
+ end, ?DELETED).
+
+
+body_size_test() ->
+ lists:foreach(fun({Size, Body}) ->
+ ?assertEqual(2 + Size, fabric2_util:rev_size(#doc{body = Body}))
+ end, ?BODIES).
+
+
+att_names_test() ->
+ lists:foreach(fun({Size, AttName}) ->
+ Att = mk_att(AttName, <<>>, <<>>, false),
+ Doc = #doc{atts = [Att]},
+ ?assertEqual(4 + Size, fabric2_util:rev_size(Doc))
+ end, ?ATT_NAMES).
+
+
+att_types_test() ->
+ lists:foreach(fun({Size, AttType}) ->
+ Att = mk_att(<<"foo">>, AttType, <<>>, false),
+ Doc = #doc{atts = [Att]},
+ ?assertEqual(7 + Size, fabric2_util:rev_size(Doc))
+ end, ?ATT_TYPES).
+
+
+att_bodies_test() ->
+ lists:foreach(fun({Size, AttBody}) ->
+ Att1 = mk_att(<<"foo">>, <<>>, AttBody, false),
+ Doc1 = #doc{atts = [Att1]},
+ ?assertEqual(7 + Size, fabric2_util:rev_size(Doc1)),
+
+ Att2 = mk_att(<<"foo">>, <<>>, AttBody, true),
+ Doc2 = #doc{atts = [Att2]},
+ ?assertEqual(7 + 16 + Size, fabric2_util:rev_size(Doc2))
+ end, ?ATT_BODIES).
+
+
+att_headers_test() ->
+ lists:foreach(fun({Size, AttHeaders}) ->
+ Att = mk_att(<<"foo">>, <<>>, <<>>, false, AttHeaders),
+ Doc = #doc{atts = [Att]},
+ ?assertEqual(7 + Size, fabric2_util:rev_size(Doc))
+ end, ?ATT_HEADERS).
+
+
+combinatorics_test() ->
+ Elements = [
+ {?DOC_IDS, fun(Doc, DocId) -> Doc#doc{id = DocId} end},
+ {?REV_STARTS, fun(Doc, RevStart) ->
+ #doc{revs = {_, RevIds}} = Doc,
+ Doc#doc{revs = {RevStart, RevIds}}
+ end},
+ {?REVS, fun(Doc, Rev) ->
+ #doc{revs = {Start, _}} = Doc,
+ Doc#doc{revs = {Start, [Rev]}}
+ end},
+ {?DELETED, fun(Doc, Deleted) -> Doc#doc{deleted = Deleted} end},
+ {?BODIES, fun(Doc, Body) -> Doc#doc{body = Body} end}
+ ],
+ combine(Elements, 0, #doc{}).
+
+
+combine([], TotalSize, Doc) ->
+ ?assertEqual(TotalSize, fabric2_util:rev_size(Doc));
+
+combine([{Elems, UpdateFun} | Rest], TotalSize, Doc) ->
+ lists:foreach(fun({Size, Elem}) ->
+ combine(Rest, TotalSize + Size, UpdateFun(Doc, Elem))
+ end, Elems).
+
+
+random_docs_test() ->
+ lists:foreach(fun(_) ->
+ {DocIdSize, DocId} = choose(?DOC_IDS),
+ {RevStartSize, RevStart} = choose(?REV_STARTS),
+ {RevSize, Rev} = choose(?REVS),
+ {DeletedSize, Deleted} = choose(?DELETED),
+ {BodySize, Body} = choose(?BODIES),
+ NumAtts = choose([0, 1, 2, 5]),
+ {Atts, AttSize} = lists:mapfoldl(fun(_, Acc) ->
+ {S, A} = random_att(),
+ {A, Acc + S}
+ end, 0, lists:seq(1, NumAtts)),
+ Doc = #doc{
+ id = DocId,
+ revs = {RevStart, [Rev]},
+ deleted = Deleted,
+ body = Body,
+ atts = Atts
+ },
+ Expect = lists:sum([
+ DocIdSize,
+ RevStartSize,
+ RevSize,
+ DeletedSize,
+ BodySize,
+ AttSize
+ ]),
+ ?assertEqual(Expect, fabric2_util:rev_size(Doc))
+ end, lists:seq(1, ?NUM_RANDOM_TESTS)).
+
+
+random_att() ->
+ {NameSize, Name} = choose(?ATT_NAMES),
+ {TypeSize, Type} = choose(?ATT_TYPES),
+ {BodySize, Body} = choose(?ATT_BODIES),
+ {Md5Size, AddMd5} = choose([{0, false}, {16, true}]),
+ {HdrSize, Headers} = choose(?ATT_HEADERS),
+ AttSize = lists:sum([NameSize, TypeSize, BodySize, Md5Size, HdrSize]),
+ {AttSize, mk_att(Name, Type, Body, AddMd5, Headers)}.
+
+
+mk_att(Name, Type, Data, AddMd5) ->
+ mk_att(Name, Type, Data, AddMd5, undefined).
+
+mk_att(Name, Type, Data, AddMd5, Headers) ->
+ Md5 = if not AddMd5 -> <<>>; true ->
+ erlang:md5(Data)
+ end,
+ couch_att:new([
+ {name, Name},
+ {type, Type},
+ {att_len, size(Data)},
+ {data, Data},
+ {encoding, identity},
+ {md5, Md5},
+ {headers, Headers}
+ ]).
+
+
+choose(Options) ->
+ Pos = rand:uniform(length(Options)),
+ lists:nth(Pos, Options).
[couchdb] 07/13: fixup: add db size tracking
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a50fbfa9635f71d43da0403667370485f84d8d6b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:06 2020 -0600
fixup: add db size tracking
---
src/fabric/src/fabric2_db.erl | 7 +++--
src/fabric/src/fabric2_fdb.erl | 63 +++++++++++++++++++++++------------------
src/fabric/src/fabric2_util.erl | 21 ++++++++++++++
3 files changed, 61 insertions(+), 30 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 26aad75..4528194 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1417,13 +1417,14 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
NewRevInfo = #{
winner => undefined,
+ exists => false,
deleted => NewDeleted,
rev_id => {NewRevPos, NewRev},
rev_path => NewRevPath,
sequence => undefined,
branch_count => undefined,
att_hash => fabric2_util:hash_atts(Atts),
- rev_size => null
+ rev_size => fabric2_util:rev_size(Doc4)
},
% Gather the list of possible winnig revisions
@@ -1474,6 +1475,7 @@ update_doc_replicated(Db, Doc0, _Options) ->
DocRevInfo0 = #{
winner => undefined,
+ exists => false,
deleted => Deleted,
rev_id => {RevPos, Rev},
rev_path => RevPath,
@@ -1520,7 +1522,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
Doc2 = prep_and_validate(Db, Doc1, PrevRevInfo),
Doc3 = flush_doc_atts(Db, Doc2),
DocRevInfo2 = DocRevInfo1#{
- atts_hash => fabric2_util:hash_atts(Doc3#doc.atts)
+ atts_hash => fabric2_util:hash_atts(Doc3#doc.atts),
+ rev_size => fabric2_util:rev_size(Doc3)
},
% Possible winners are the previous winner and
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index f447e93..8d8616d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -632,7 +632,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Doc body
- {ok, RevSize} = write_doc_body(Db, Doc),
+ ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -663,8 +663,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Revision tree
NewWinner = NewWinner0#{
- winner := true,
- rev_size := RevSize
+ winner := true
},
NewRevId = maps:get(rev_id, NewWinner),
@@ -767,13 +766,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
end,
% Update database size
- SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
- Acc + case maps:get(rev_size, RI, null) of
- null -> 0;
- Size -> Size
- end
- end, 0, ToRemove),
- incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+ AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
+ RemSize = sum_rem_rev_sizes(ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, AddSize - RemSize),
ok.
@@ -1091,11 +1086,10 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
- {Rows, RevSize} = doc_to_fdb(Db, Doc),
+ Rows = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, Rows),
- {ok, RevSize}.
+ end, Rows).
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1208,6 +1202,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
{_RevFormat, Sequence, BranchCount, RevPath, AttHash, RevSize} = Val,
#{
winner => true,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1222,6 +1217,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
{_RevFormat, RevPath, AttHash, RevSize} = Val,
#{
winner => false,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1240,11 +1236,11 @@ fdb_to_revinfo(Key, {0, RPath}) ->
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, Seq, BCount, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val).
@@ -1271,19 +1267,7 @@ doc_to_fdb(Db, #doc{} = Doc) ->
{{Key, Chunk}, ChunkId + 1}
end, 0, Chunks),
- % Calculate the size of this revision
- TotalSize = lists:sum([
- size(Id),
- size(erlfdb_tuple:pack({Start})),
- size(Rev),
- 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
- couch_ejson_size:encoded_size(Body),
- lists:foldl(fun(Att, Acc) ->
- couch_att:external_size(Att) + Acc
- end, 0, Atts)
- ]),
-
- {Rows, TotalSize}.
+ Rows.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1388,6 +1372,29 @@ fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
fdb_to_local_doc(Db, DocId, Rev, Rows).
+sum_add_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := Exists,
+ rev_size := Size
+ } = RI,
+ case Exists of
+ true -> Acc;
+ false -> Size + Acc
+ end
+ end, 0, RevInfos).
+
+
+sum_rem_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := true,
+ rev_size := Size
+ } = RI,
+ Size + Acc
+ end, 0, RevInfos).
+
+
chunkify_binary(Data) ->
case Data of
<<>> ->
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 4e2e2d7..0f75390 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -17,6 +17,7 @@
revinfo_to_revs/1,
revinfo_to_path/1,
sort_revinfos/1,
+ rev_size/1,
seq_zero_vs/0,
seq_max_vs/0,
@@ -78,6 +79,26 @@ rev_sort_key(#{} = RevInfo) ->
{not Deleted, RevPos, Rev}.
+rev_size(#doc{} = Doc) ->
+ #doc{
+ id = Id,
+ revs = {Start, [Rev | _]},
+ body = Body,
+ atts = Atts
+ } = Doc,
+
+ lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]).
+
+
seq_zero_vs() ->
{versionstamp, 0, 0, 0}.
[couchdb] 01/13: Track a database level view size rollup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b2b5208fd151a92d1dd04e76f72f875a74460323
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 10:24:36 2019 -0600
Track a database level view size rollup
This way we can expose the total view size for a database in the dbinfo
JSON blob.
---
src/couch_views/src/couch_views_fdb.erl | 17 ++++++++++++++--
src/fabric/src/fabric2_fdb.erl | 36 ++++++++++++++++-----------------
2 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 98cff46..5edaa3a 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -272,8 +272,16 @@ update_kv_size(TxDb, Sig, ViewId, Increment) ->
tx := Tx,
db_prefix := DbPrefix
} = TxDb,
- Key = kv_size_key(DbPrefix, Sig, ViewId),
- erlfdb:add(Tx, Key, Increment).
+
+ % Track a view specific size for calls to
+ % GET /dbname/_design/doc/_info`
+ IdxKey = kv_size_key(DbPrefix, Sig, ViewId),
+ erlfdb:add(Tx, IdxKey, Increment),
+
+ % Track a database level rollup for calls to
+ % GET /dbname
+ DbKey = db_kv_size_key(DbPrefix),
+ erlfdb:add(Tx, DbKey, Increment).
seq_key(DbPrefix, Sig) ->
@@ -291,6 +299,11 @@ kv_size_key(DbPrefix, Sig, ViewId) ->
erlfdb_tuple:pack(Key, DbPrefix).
+db_kv_size_key(DbPrefix) ->
+ Key = {?DB_STATS, <<"sizes">>, <<"views">>},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 6abe1f6..8bfbb74 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -174,11 +174,16 @@ create(#{} = Db0, Options) ->
{?DB_STATS, <<"doc_del_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_design_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_local_count">>, ?uint2bin(0)},
- {?DB_STATS, <<"size">>, ?uint2bin(2)}
+ {?DB_STATS, <<"sizes">>, <<"external">>, ?uint2bin(2)},
+ {?DB_STATS, <<"sizes">>, <<"views">>, ?uint2bin(0)}
],
- lists:foreach(fun({P, K, V}) ->
- Key = erlfdb_tuple:pack({P, K}, DbPrefix),
- erlfdb:set(Tx, Key, V)
+ lists:foreach(fun
+ ({P, K, V}) ->
+ Key = erlfdb_tuple:pack({P, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V);
+ ({P, S, K, V}) ->
+ Key = erlfdb_tuple:pack({P, S, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V)
end, Defaults),
UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}),
@@ -348,26 +353,21 @@ get_info(#{} = Db) ->
end,
CProp = {update_seq, RawSeq},
- MProps = lists:flatmap(fun({K, V}) ->
+ MProps = lists:foldl(fun({K, V}, Acc) ->
case erlfdb_tuple:unpack(K, DbPrefix) of
{?DB_STATS, <<"doc_count">>} ->
- [{doc_count, ?bin2uint(V)}];
+ [{doc_count, ?bin2uint(V)} | Acc];
{?DB_STATS, <<"doc_del_count">>} ->
- [{doc_del_count, ?bin2uint(V)}];
- {?DB_STATS, <<"size">>} ->
+ [{doc_del_count, ?bin2uint(V)} | Acc];
+ {?DB_STATS, <<"sizes">>, Name} ->
Val = ?bin2uint(V),
- [
- {other, {[{data_size, Val}]}},
- {sizes, {[
- {active, 0},
- {external, Val},
- {file, 0}
- ]}}
- ];
+ {_, {Sizes}} = lists:keyfind(sizes, 1, Acc),
+ NewSizes = lists:keystore(Name, 1, Sizes, {Name, Val}),
+ lists:keystore(sizes, 1, Acc, {sizes, {NewSizes}});
{?DB_STATS, _} ->
- []
+ Acc
end
- end, erlfdb:wait(MetaFuture)),
+ end, [{sizes, {[]}}], erlfdb:wait(MetaFuture)),
[CProp | MProps].
[couchdb] 13/13: fixup: add size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ae8d939afaf5be314c4516ff3523d165f617774b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Feb 12 17:18:24 2020 -0600
fixup: add size tests
---
src/fabric/test/fabric2_db_size_tests.erl | 490 ++++++++++++++++++++++--------
1 file changed, 356 insertions(+), 134 deletions(-)
diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
index f9d514d..e963670 100644
--- a/src/fabric/test/fabric2_db_size_tests.erl
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -12,6 +12,9 @@
-module(fabric2_db_size_tests).
+-export([
+ random_body/0
+]).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch/include/couch_eunit.hrl").
@@ -45,18 +48,30 @@ db_size_test_() ->
fun setup/0,
fun cleanup/1,
with([
- ?TDEF(empty_size),
?TDEF(new_doc),
?TDEF(edit_doc),
- ?TDEF(del_doc),
- ?TDEF(conflicted_doc),
- ?TDEF(del_winner),
- ?TDEF(del_conflict)
+ ?TDEF(delete_doc),
+ ?TDEF(create_conflict),
+ ?TDEF(replicate_new_winner),
+ ?TDEF(replicate_deep_deleted),
+ ?TDEF(delete_winning_revision),
+ ?TDEF(delete_conflict_revision)
])
}
}.
+% TODO:
+% replicate existing revision
+% replicate with attachment
+% replicate removing attachment
+% replicate reusing attachment
+% replicate adding attachment with stub
+% for each, replicate to winner vs non-winner
+% for each, replicate extending winner, vs extending conflict vs new branch
+
+
+
setup() ->
Ctx = test_util:start_couch([fabric]),
{ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
@@ -68,161 +83,288 @@ cleanup({Db, Ctx}) ->
test_util:stop_couch(Ctx).
-empty_size({Db, _}) ->
- ?assertEqual(2, db_size(Db)).
-
-
new_doc({Db, _}) ->
- % UUID doc id: 32
- % Revision: 2 + 16
- % Deleted: 1
- % Body: {} = 2
- ?DIFF(Db, 53, fun() ->
- create_doc(Db)
- end).
+ Actions = [
+ {create, #{tgt => rev1}}
+ ],
+ check(Db, Actions).
edit_doc({Db, _}) ->
- DocId = fabric2_util:uuid(),
- {ok, RevId1} = ?DIFF(Db, 53, fun() ->
- create_doc(Db, DocId)
- end),
- % {} -> {"foo":"bar"} = 13 - 2
- {ok, RevId2} = ?DIFF(Db, 11, fun() ->
- update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
- end),
- ?DIFF(Db, -11, fun() ->
- update_doc(Db, DocId, RevId2)
- end).
-
-
-del_doc({Db, _}) ->
- DocId = fabric2_util:uuid(),
- {ok, RevId} = ?DIFF(Db, 64, fun() ->
- create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
- end),
- % The change here is -11 becuase we're going from
- % {"foo":"bar"} == 13 bytes to {} == 2 bytes.
- % I.e., 2 - 13 == -11
- ?DIFF(Db, -11, fun() ->
- delete_doc(Db, DocId, RevId)
- end).
-
-
-% need to check both new conflict is new winner
-% and that new conflict is not a winner and that
-% the sizes don't interfere which should be doable
-% with different sized bodies.
-
-conflicted_doc({Db, _}) ->
- DocId = fabric2_util:uuid(),
- {ok, RevId1} = ?DIFF(Db, 64, fun() ->
- create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
- end),
- ?DIFF(Db, 64, fun() ->
- create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
- end).
-
-
-del_winner({Db, _}) ->
- DocId = fabric2_util:uuid(),
- {ok, RevId1} = ?DIFF(Db, 64, fun() ->
- create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
- end),
- {ok, RevId2} = ?DIFF(Db, 64, fun() ->
- create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
- end),
- [_ConflictRev, WinnerRev] = lists:sort([RevId1, RevId2]),
- ?DIFF(Db, -11, fun() ->
- {ok, _RevId3} = delete_doc(Db, DocId, WinnerRev),
- ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
- end).
-
-
-del_conflict({Db, _}) ->
- DocId = fabric2_util:uuid(),
- {ok, RevId1} = ?DIFF(Db, 64, fun() ->
- create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
- end),
- {ok, RevId2} = ?DIFF(Db, 64, fun() ->
- create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
- end),
- [ConflictRev, _WinnerRev] = lists:sort([RevId1, RevId2]),
- ?DIFF(Db, -11, fun() ->
- {ok, _RevId3} = delete_doc(Db, DocId, ConflictRev),
- ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
- end).
+ Actions = [
+ {create, #{tgt => rev1}},
+ {update, #{src => rev1, tgt => rev2}}
+ ],
+ check(Db, Actions).
+
+
+delete_doc({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1}},
+ {delete, #{src => rev1, tgt => rev2}}
+ ],
+ check(Db, Actions).
+
+
+create_conflict({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1}},
+ {replicate, #{tgt => rev2}}
+ ],
+ check(Db, Actions).
+
+
+replicate_new_winner({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1}},
+ {replicate, #{tgt => rev2, depth => 3}}
+ ],
+ check(Db, Actions).
+
+
+replicate_deep_deleted({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1, depth => 2}},
+ {replicate, #{tgt => rev2, depth => 5, deleted => true}}
+ ],
+ check(Db, Actions).
+
+
+delete_winning_revision({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1}},
+ {replicate, #{tgt => rev2}},
+ {delete, #{src => {min, [rev1, rev2]}, tgt => rev3}}
+ ],
+ check(Db, Actions).
+
+
+delete_conflict_revision({Db, _}) ->
+ Actions = [
+ {create, #{tgt => rev1}},
+ {replicate, #{tgt => rev2}},
+ {delete, #{src => {max, [rev1, rev2]}, tgt => rev3}}
+ ],
+ check(Db, Actions).
+
+
+check(Db, Actions) ->
+ InitSt = #{
+ doc_id => couch_uuids:random(),
+ revs => #{},
+ atts => #{},
+ size => db_size(Db)
+ },
+ lists:foldl(fun({Action, Opts}, StAcc) ->
+ case Action of
+ create -> create_doc(Db, Opts, StAcc);
+ update -> update_doc(Db, Opts, StAcc);
+ delete -> delete_doc(Db, Opts, StAcc);
+ replicate -> replicate_doc(Db, Opts, StAcc)
+ end
+ end, InitSt, Actions).
+
+
+create_doc(Db, Opts, St) ->
+ #{
+ doc_id := DocId,
+ revs := Revs,
+ size := InitDbSize
+ } = St,
+
+ ?assert(maps:is_key(tgt, Opts)),
+
+ Tgt = maps:get(tgt, Opts),
+ Depth = maps:get(depth, Opts, 1),
+
+ ?assert(not maps:is_key(Tgt, Revs)),
+ ?assert(Depth >= 1),
+
+ InitDoc = #doc{id = DocId},
+ FinalDoc = lists:foldl(fun(_, Doc0) ->
+ #doc{
+ revs = {_OldStart, OldRevs}
+ } = Doc1 = randomize_doc(Doc0),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1),
+ Doc1#doc{revs = {Pos, [Rev | OldRevs]}}
+ end, InitDoc, lists:seq(1, Depth)),
+
+ FinalDocSize = fabric2_util:rev_size(FinalDoc),
+ FinalDbSize = db_size(Db),
+
+ ?assertEqual(FinalDbSize - InitDbSize, FinalDocSize),
+
+ St#{
+ revs := maps:put(Tgt, FinalDoc, Revs),
+ size := FinalDbSize
+ }.
-% replicate with attachment
-% replicate removing attachment
-% replicate reusing attachment
-% replicate adding attachment with stub
-% for each, replicate to winner vs non-winner
-% for each, replicate extending winner, vs extending conflict vs new branch
+update_doc(Db, Opts, St) ->
+ #{
+ revs := Revs,
+ size := InitDbSize
+ } = St,
+ ?assert(maps:is_key(src, Opts)),
+ ?assert(maps:is_key(tgt, Opts)),
+ Src = pick_rev(Revs, maps:get(src, Opts)),
+ Tgt = maps:get(tgt, Opts),
+ Depth = maps:get(depth, Opts, 1),
-create_doc(Db) ->
- create_doc(Db, fabric2_util:uuid()).
+ ?assert(maps:is_key(Src, Revs)),
+ ?assert(not maps:is_key(Tgt, Revs)),
+ ?assert(Depth >= 1),
+ InitDoc = maps:get(Src, Revs),
+ FinalDoc = lists:foldl(fun(_, Doc0) ->
+ #doc{
+ revs = {_OldStart, OldRevs}
+ } = Doc1 = randomize_doc(Doc0),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1),
+ Doc1#doc{revs = {Pos, [Rev | OldRevs]}}
+ end, InitDoc, lists:seq(1, Depth)),
-create_doc(Db, DocId) when is_binary(DocId) ->
- create_doc(Db, DocId, {[]});
-create_doc(Db, {Props} = Body) when is_list(Props) ->
- create_doc(Db, fabric2_util:uuid(), Body).
+ InitDocSize = fabric2_util:rev_size(InitDoc),
+ FinalDocSize = fabric2_util:rev_size(FinalDoc),
+ FinalDbSize = db_size(Db),
+ ?assertEqual(FinalDbSize - InitDbSize, FinalDocSize - InitDocSize),
-create_doc(Db, DocId, Body) ->
- Doc = #doc{
- id = DocId,
- body = Body
- },
- fabric2_db:update_doc(Db, Doc).
+ St#{
+ revs := maps:put(Tgt, FinalDoc, Revs),
+ size := FinalDbSize
+ }.
-create_conflict(Db, DocId, RevId) ->
- create_conflict(Db, DocId, RevId, {[]}).
+delete_doc(Db, Opts, St) ->
+ #{
+ revs := Revs,
+ size := InitDbSize
+ } = St,
+ ?assert(maps:is_key(src, Opts)),
+ ?assert(maps:is_key(tgt, Opts)),
-create_conflict(Db, DocId, RevId, Body) ->
- {Pos, _} = RevId,
- % Only keep the first 16 bytes of the UUID
- % so that we match the normal sized revs
- <<NewRev:16/binary, _/binary>> = fabric2_util:uuid(),
- Doc = #doc{
- id = DocId,
- revs = {Pos, [NewRev]},
- body = Body
- },
- fabric2_db:update_doc(Db, Doc, [replicated_changes]).
+ Src = pick_rev(Revs, maps:get(src, Opts)),
+ Tgt = maps:get(tgt, Opts),
+ ?assert(maps:is_key(Src, Revs)),
+ ?assert(not maps:is_key(Tgt, Revs)),
-update_doc(Db, DocId, RevId) ->
- update_doc(Db, DocId, RevId, {[]}).
+ InitDoc = maps:get(Src, Revs),
+ #doc{
+ revs = {_OldStart, OldRevs}
+ } = UpdateDoc = randomize_deleted_doc(InitDoc),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, UpdateDoc),
-update_doc(Db, DocId, {Pos, Rev}, Body) ->
- Doc = #doc{
- id = DocId,
- revs = {Pos, [Rev]},
- body = Body
+ FinalDoc = UpdateDoc#doc{
+ revs = {Pos, [Rev | OldRevs]}
},
- fabric2_db:update_doc(Db, Doc).
+ InitDocSize = fabric2_util:rev_size(InitDoc),
+ FinalDocSize = fabric2_util:rev_size(FinalDoc),
+ FinalDbSize = db_size(Db),
-delete_doc(Db, DocId, RevId) ->
- delete_doc(Db, DocId, RevId, {[]}).
+ ?assertEqual(FinalDbSize - InitDbSize, FinalDocSize - InitDocSize),
+ St#{
+ revs := maps:put(Tgt, FinalDoc, Revs),
+ size := FinalDbSize
+ }.
-delete_doc(Db, DocId, {Pos, Rev}, Body) ->
- Doc = #doc{
- id = DocId,
- revs = {Pos, [Rev]},
- deleted = true,
- body = Body
- },
- fabric2_db:update_doc(Db, Doc).
+
+replicate_doc(Db, Opts, St) ->
+ #{
+ doc_id := DocId,
+ revs := Revs,
+ size := InitDbSize
+ } = St,
+
+ ?assert(maps:is_key(tgt, Opts)),
+
+ Src = pick_rev(Revs, maps:get(src, Opts, undefined)),
+ Tgt = maps:get(tgt, Opts),
+ Deleted = maps:get(deleted, Opts, false),
+ Depth = maps:get(depth, Opts, 1),
+
+ if Src == undefined -> ok; true ->
+ ?assert(maps:is_key(Src, Revs))
+ end,
+ ?assert(not maps:is_key(Tgt, Revs)),
+ ?assert(is_boolean(Deleted)),
+ ?assert(Depth >= 1),
+
+ InitDoc1 = maps:get(Src, Revs, #doc{id = DocId}),
+ InitDoc2 = case Deleted of
+ true -> randomize_deleted_doc(InitDoc1);
+ false -> randomize_doc(InitDoc1)
+ end,
+ FinalDoc = lists:foldl(fun(_, Doc0) ->
+ #doc{
+ revs = {RevStart, RevIds}
+ } = Doc0,
+ NewRev = crypto:strong_rand_bytes(16),
+ Doc0#doc{
+ revs = {RevStart + 1, [NewRev | RevIds]}
+ }
+ end, InitDoc2, lists:seq(1, Depth)),
+
+ {ok, _} = fabric2_db:update_doc(Db, FinalDoc, [replicated_changes]),
+
+ InitDocSize = fabric2_util:rev_size(InitDoc1),
+ FinalDocSize = fabric2_util:rev_size(FinalDoc),
+ FinalDbSize = db_size(Db),
+
+ SizeChange = case Src of
+ undefined -> FinalDocSize;
+ _ -> FinalDocSize - InitDocSize
+ end,
+ ?assertEqual(FinalDbSize - InitDbSize, SizeChange),
+
+ St#{
+ revs := maps:put(Tgt, FinalDoc, Revs),
+ size := FinalDbSize
+ }.
+
+
+pick_rev(_Revs, Rev) when is_atom(Rev) ->
+ Rev;
+pick_rev(Revs, {Op, RevList}) when Op == min; Op == max ->
+ ChooseFrom = lists:map(fun(Rev) ->
+ #doc{
+ revs = {S, [R | _]},
+ deleted = Deleted
+ } = maps:get(Rev, Revs),
+ #{
+ deleted => Deleted,
+ rev_id => {S, R},
+ name => Rev
+ }
+ end, RevList),
+ Sorted = fabric2_util:sort_revinfos(ChooseFrom),
+ RetRev = case Op of
+ min -> lists:last(Sorted);
+ max -> hd(Sorted)
+ end,
+ maps:get(name, RetRev).
+
+
+randomize_doc(#doc{} = Doc) ->
+ Doc#doc{
+ deleted = false,
+ body = random_body()
+ }.
+
+
+randomize_deleted_doc(Doc) ->
+ NewDoc = case rand:uniform() < 0.05 of
+ true -> randomize_doc(Doc);
+ false -> Doc#doc{body = {[]}}
+ end,
+ NewDoc#doc{deleted = true}.
db_size(Info) when is_list(Info) ->
@@ -232,3 +374,83 @@ db_size(Info) when is_list(Info) ->
db_size(Db) when is_map(Db) ->
{ok, Info} = fabric2_db:get_db_info(Db),
db_size(Info).
+
+
+
+-define(MAX_JSON_ELEMENTS, 5).
+-define(MAX_STRING_LEN, 10).
+-define(MAX_INT, 4294967296).
+
+
+random_body() ->
+ Elems = rand:uniform(?MAX_JSON_ELEMENTS),
+ {Obj, _} = random_json_object(Elems),
+ Obj.
+
+
+random_json(MaxElems) ->
+ case choose([object, array, terminal]) of
+ object -> random_json_object(MaxElems);
+ array -> random_json_array(MaxElems);
+ terminal -> {random_json_terminal(), MaxElems}
+ end.
+
+
+random_json_object(MaxElems) ->
+ NumKeys = rand:uniform(MaxElems + 1) - 1,
+ {Props, RemElems} = lists:mapfoldl(fun(_, Acc1) ->
+ {Value, Acc2} = random_json(Acc1),
+ {{random_json_string(), Value}, Acc2}
+ end, MaxElems - NumKeys, lists:seq(1, NumKeys)),
+ {{Props}, RemElems}.
+
+
+random_json_array(MaxElems) ->
+ NumItems = rand:uniform(MaxElems + 1) - 1,
+ lists:mapfoldl(fun(_, Acc1) ->
+ random_json(Acc1)
+ end, MaxElems - NumItems, lists:seq(1, NumItems)).
+
+
+random_json_terminal() ->
+ case choose([null, true, false, number, string]) of
+ null -> null;
+ true -> true;
+ false -> false;
+ number -> random_json_number();
+ string -> random_json_string()
+ end.
+
+
+random_json_number() ->
+ AbsValue = case choose([integer, double]) of
+ integer -> rand:uniform(?MAX_INT);
+ double -> rand:uniform() * rand:uniform()
+ end,
+ case choose([pos, neg]) of
+ pos -> AbsValue;
+ neg -> -1 * AbsValue
+ end.
+
+
+random_json_string() ->
+ Alphabet = [
+ $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m,
+ $n, $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z,
+ $A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M,
+ $N, $O, $P, $Q, $R, $S, $T, $U, $V, $W, $Y, $X, $Z,
+ $1, $2, $3, $4, $5, $6, $7, $8, $9, $0,
+ $!, $@, $#, $$, $%, $^, $&, $*, $(, $),
+ $ , ${, $}, $[, $], $", $', $-, $_, $+, $=, $,, $.,
+ $\x{1}, $\x{a2}, $\x{20ac}, $\x{10348}
+ ],
+ Len = rand:uniform(?MAX_STRING_LEN) - 1,
+ Str = lists:map(fun(_) ->
+ choose(Alphabet)
+ end, lists:seq(1, Len)),
+ unicode:characters_to_binary(Str).
+
+
+choose(Options) ->
+ Pos = rand:uniform(length(Options)),
+ lists:nth(Pos, Options).
\ No newline at end of file
[couchdb] 06/13: Add database size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 04cbdbd62d0e8a3c3eba0a06ce3e468bf3afe62c
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 6 14:59:30 2019 -0600
Add database size tests
---
src/fabric/test/fabric2_db_size_tests.erl | 198 ++++++++++++++++++++++++++++++
1 file changed, 198 insertions(+)
diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
new file mode 100644
index 0000000..fc95e0e
--- /dev/null
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -0,0 +1,198 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_db_size_tests).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("fabric2_test.hrl").
+
+
+db_size_test_() ->
+ {
+ "Test document CRUD operations",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ with([
+ ?TDEF(empty_size),
+ ?TDEF(new_doc),
+ ?TDEF(edit_doc),
+ ?TDEF(del_doc),
+ ?TDEF(conflicted_doc),
+ ?TDEF(del_conflict)
+ ])
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([fabric]),
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ {Db, Ctx}.
+
+
+cleanup({Db, Ctx}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []),
+ test_util:stop_couch(Ctx).
+
+
+empty_size({Db, _}) ->
+ ?assertEqual(2, db_size(Db)).
+
+
+new_doc({Db, _}) ->
+ increases(Db, fun() ->
+ create_doc(Db)
+ end).
+
+
+edit_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId)
+ end),
+ {ok, RevId2} = increases(Db, fun() ->
+ update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ decreases(Db, fun() ->
+ update_doc(Db, DocId, RevId2)
+ end).
+
+
+del_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ % The change here is -11 becuase we're going from
+ % {"foo":"bar"} == 13 bytes to {} == 2 bytes.
+ % I.e., 2 - 13 == -11
+ diff(Db, fun() ->
+ delete_doc(Db, DocId, RevId)
+ end, -11).
+
+
+conflicted_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ Before = db_size(Db),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ Between = db_size(Db),
+ increases(Db, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ After = db_size(Db),
+ ?assertEqual(After - Between, Between - Before).
+
+
+del_conflict({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ {ok, RevId2} = increases(Db, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ decreases(Db, fun() ->
+ {ok, RevId3} = delete_doc(Db, DocId, RevId2),
+ ?debugFmt("~p ~p ~p", [RevId1, RevId2, RevId3])
+ end).
+
+
+create_doc(Db) ->
+ create_doc(Db, fabric2_util:uuid()).
+
+
+create_doc(Db, DocId) when is_binary(DocId) ->
+ create_doc(Db, DocId, {[]});
+create_doc(Db, {Props} = Body) when is_list(Props) ->
+ create_doc(Db, fabric2_util:uuid(), Body).
+
+
+create_doc(Db, DocId, Body) ->
+ Doc = #doc{
+ id = DocId,
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+create_conflict(Db, DocId, RevId) ->
+ create_conflict(Db, DocId, RevId, {[]}).
+
+
+create_conflict(Db, DocId, RevId, Body) ->
+ {Pos, _} = RevId,
+ % Only keep the first 16 bytes of the UUID
+ % so that we match the normal sized revs
+ <<NewRev:16/binary, _/binary>> = fabric2_util:uuid(),
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [NewRev]},
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc, [replicated_changes]).
+
+
+update_doc(Db, DocId, RevId) ->
+ update_doc(Db, DocId, RevId, {[]}).
+
+
+update_doc(Db, DocId, {Pos, Rev}, Body) ->
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [Rev]},
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+delete_doc(Db, DocId, RevId) ->
+ delete_doc(Db, DocId, RevId, {[]}).
+
+
+delete_doc(Db, DocId, {Pos, Rev}, Body) ->
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [Rev]},
+ deleted = true,
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+constant(Db, Fun) -> check(Db, Fun, fun erlang:'=='/2).
+increases(Db, Fun) -> check(Db, Fun, fun erlang:'>'/2).
+decreases(Db, Fun) -> check(Db, Fun, fun erlang:'<'/2).
+diff(Db, Fun, Change) -> check(Db, Fun, fun(A, B) -> (A - B) == Change end).
+
+check(Db, Fun, Cmp) ->
+ Before = db_size(Db),
+ Result = Fun(),
+ After = db_size(Db),
+ ?debugFmt("~p :: ~p ~p", [erlang:fun_info(Cmp), After, Before]),
+ ?assert(Cmp(After, Before)),
+ Result.
+
+
+db_size(Info) when is_list(Info) ->
+ {sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
+ {<<"external">>, External} = lists:keyfind(<<"external">>, 1, Sizes),
+ External;
+db_size(Db) when is_map(Db) ->
+ {ok, Info} = fabric2_db:get_db_info(Db),
+ db_size(Info).
[couchdb] 09/13: fixup: db size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 052610cc700c7429bae2dd58f52ee29fd48e278b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:50 2020 -0600
fixup: db size tests
---
src/fabric/test/fabric2_db_size_tests.erl | 100 ++++++++++++++++++++----------
1 file changed, 68 insertions(+), 32 deletions(-)
diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
index fc95e0e..f9d514d 100644
--- a/src/fabric/test/fabric2_db_size_tests.erl
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -19,6 +19,24 @@
-include("fabric2_test.hrl").
+-define(DIFF(Db, Change, Fun), begin
+ ((fun() ->
+ __Before = db_size(Db),
+ __Result = Fun(),
+ __After = db_size(Db),
+ ?debugFmt("~p - ~p == ~p ?= ~p", [
+ __After,
+ __Before,
+ __After - __Before,
+ Change
+ ]),
+ ?assertEqual(Change, __After - __Before),
+ __Result
+ end)())
+ end
+).
+
+
db_size_test_() ->
{
"Test document CRUD operations",
@@ -32,6 +50,7 @@ db_size_test_() ->
?TDEF(edit_doc),
?TDEF(del_doc),
?TDEF(conflicted_doc),
+ ?TDEF(del_winner),
?TDEF(del_conflict)
])
}
@@ -54,65 +73,96 @@ empty_size({Db, _}) ->
new_doc({Db, _}) ->
- increases(Db, fun() ->
+ % UUID doc id: 32
+ % Revision: 2 + 16
+ % Deleted: 1
+ % Body: {} = 2
+ ?DIFF(Db, 53, fun() ->
create_doc(Db)
end).
edit_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 53, fun() ->
create_doc(Db, DocId)
end),
- {ok, RevId2} = increases(Db, fun() ->
+ % {} -> {"foo":"bar"} = 13 - 2
+ {ok, RevId2} = ?DIFF(Db, 11, fun() ->
update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- decreases(Db, fun() ->
+ ?DIFF(Db, -11, fun() ->
update_doc(Db, DocId, RevId2)
end).
del_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId} = increases(Db, fun() ->
+ {ok, RevId} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
% The change here is -11 becuase we're going from
% {"foo":"bar"} == 13 bytes to {} == 2 bytes.
% I.e., 2 - 13 == -11
- diff(Db, fun() ->
+ ?DIFF(Db, -11, fun() ->
delete_doc(Db, DocId, RevId)
- end, -11).
+ end).
+% need to check both new conflict is new winner
+% and that new conflict is not a winner and that
+% the sizes don't interfere which should be doable
+% with different sized bodies.
+
conflicted_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- Before = db_size(Db),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ ?DIFF(Db, 64, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end).
+
+
+del_winner({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
- Between = db_size(Db),
- increases(Db, fun() ->
+ {ok, RevId2} = ?DIFF(Db, 64, fun() ->
create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- After = db_size(Db),
- ?assertEqual(After - Between, Between - Before).
+ [_ConflictRev, WinnerRev] = lists:sort([RevId1, RevId2]),
+ ?DIFF(Db, -11, fun() ->
+ {ok, _RevId3} = delete_doc(Db, DocId, WinnerRev),
+ ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
+ end).
del_conflict({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
- {ok, RevId2} = increases(Db, fun() ->
+ {ok, RevId2} = ?DIFF(Db, 64, fun() ->
create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- decreases(Db, fun() ->
- {ok, RevId3} = delete_doc(Db, DocId, RevId2),
- ?debugFmt("~p ~p ~p", [RevId1, RevId2, RevId3])
+ [ConflictRev, _WinnerRev] = lists:sort([RevId1, RevId2]),
+ ?DIFF(Db, -11, fun() ->
+ {ok, _RevId3} = delete_doc(Db, DocId, ConflictRev),
+ ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
end).
+% replicate with attachment
+% replicate removing attachment
+% replicate reusing attachment
+% replicate adding attachment with stub
+% for each, replicate to winner vs non-winner
+% for each, replicate extending winner, vs extending conflict vs new branch
+
+
+
create_doc(Db) ->
create_doc(Db, fabric2_util:uuid()).
@@ -175,20 +225,6 @@ delete_doc(Db, DocId, {Pos, Rev}, Body) ->
fabric2_db:update_doc(Db, Doc).
-constant(Db, Fun) -> check(Db, Fun, fun erlang:'=='/2).
-increases(Db, Fun) -> check(Db, Fun, fun erlang:'>'/2).
-decreases(Db, Fun) -> check(Db, Fun, fun erlang:'<'/2).
-diff(Db, Fun, Change) -> check(Db, Fun, fun(A, B) -> (A - B) == Change end).
-
-check(Db, Fun, Cmp) ->
- Before = db_size(Db),
- Result = Fun(),
- After = db_size(Db),
- ?debugFmt("~p :: ~p ~p", [erlang:fun_info(Cmp), After, Before]),
- ?assert(Cmp(After, Before)),
- Result.
-
-
db_size(Info) when is_list(Info) ->
{sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
{<<"external">>, External} = lists:keyfind(<<"external">>, 1, Sizes),
[couchdb] 08/13: Fix doc attachment tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1cfd2a5d0e27ad5fe985c9b454f6e0c0c1c831f9
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:22 2020 -0600
Fix doc attachment tests
---
src/fabric/test/fabric2_doc_att_tests.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/fabric/test/fabric2_doc_att_tests.erl b/src/fabric/test/fabric2_doc_att_tests.erl
index 331e1a4..ac531e9 100644
--- a/src/fabric/test/fabric2_doc_att_tests.erl
+++ b/src/fabric/test/fabric2_doc_att_tests.erl
@@ -175,9 +175,9 @@ large_att({Db, _}) ->
AttData = iolist_to_binary([
<<"foobar">> || _ <- lists:seq(1, 60000)
]),
- Att1 = mk_att("long.txt", AttData),
+ Att1 = mk_att(<<"long.txt">>, AttData),
{ok, _} = create_doc(Db, DocId, [Att1]),
- ?assertEqual(#{"long.txt" => AttData}, read_atts(Db, DocId)),
+ ?assertEqual(#{<<"long.txt">> => AttData}, read_atts(Db, DocId)),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
#doc{atts = [Att2]} = Doc,
[couchdb] 04/13: Support `GET /_dbs_info` endpoint
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3d4970fec724d49c7565317a3f1d6ca95d344369
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 13:44:35 2019 -0600
Support `GET /_dbs_info` endpoint
Previously only `POST` with a list of keys was supported. The new `GET`
support just dumps all database info blobs in a single ordered response.
---
src/chttpd/src/chttpd_misc.erl | 49 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 186ec9f..b181f3c 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -157,6 +157,37 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
{ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
{ok, Acc#vacc{resp=Resp1}}.
+handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
+ ok = chttpd:verify_is_server_admin(Req),
+
+ #mrargs{
+ start_key = StartKey,
+ end_key = EndKey,
+ direction = Dir,
+ limit = Limit,
+ skip = Skip
+ } = couch_mrview_http:parse_params(Req, undefined),
+
+ Options = [
+ {start_key, StartKey},
+ {end_key, EndKey},
+ {dir, Dir},
+ {limit, Limit},
+ {skip, Skip}
+ ],
+
+ % TODO: Figure out if we can't calculate a valid
+ % ETag for this request. \xFFmetadataVersion won't
+ % work as we don't bump versions on size changes
+
+ {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, []),
+ Callback = fun dbs_info_callback/2,
+ Acc = #vacc{req = Req, resp = Resp},
+ {ok, Resp} = fabric2_db:list_dbs_info(Callback, Acc, Options),
+ case is_record(Resp, vacc) of
+ true -> {ok, Resp#vacc.resp};
+ _ -> {ok, Resp}
+ end;
handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
chttpd:validate_ctype(Req, "application/json"),
Props = chttpd:json_body_obj(Req),
@@ -188,7 +219,23 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
send_chunk(Resp, "]"),
chttpd:end_json_response(Resp);
handle_dbs_info_req(Req) ->
- send_method_not_allowed(Req, "POST").
+ send_method_not_allowed(Req, "GET,HEAD,POST").
+
+dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
+ {ok, Acc#vacc{resp = Resp1}};
+dbs_info_callback({row, Props}, #vacc{resp = Resp0} = Acc) ->
+ Prepend = couch_mrview_http:prepend_val(Acc),
+ Chunk = [Prepend, ?JSON_ENCODE({Props})],
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+ {ok, Acc#vacc{prepend = ",", resp = Resp1}};
+dbs_info_callback(complete, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, Acc#vacc{resp = Resp2}};
+dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
+ {ok, Acc#vacc{resp = Resp1}}.
handle_task_status_req(#httpd{method='GET'}=Req) ->
ok = chttpd:verify_is_server_admin(Req),
[couchdb] 02/13: Implement async API for `fabric2_fdb:get_info/1`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b8b13ec7d49fda1de21fa5c9665c312acba9eddb
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:44:34 2019 -0600
Implement async API for `fabric2_fdb:get_info/1`
---
src/fabric/src/fabric2_fdb.erl | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 8bfbb74..0e7cba8 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -29,6 +29,8 @@
list_dbs/4,
get_info/1,
+ get_info_future/2,
+ get_info_wait/1,
set_config/3,
get_stat/2,
@@ -333,7 +335,10 @@ get_info(#{} = Db) ->
tx := Tx,
db_prefix := DbPrefix
} = ensure_current(Db),
+ get_info_wait(get_info_future(Tx, DbPrefix)).
+
+get_info_future(Tx, DbPrefix) ->
{CStart, CEnd} = erlfdb_tuple:range({?DB_CHANGES}, DbPrefix),
ChangesFuture = erlfdb:get_range(Tx, CStart, CEnd, [
{streaming_mode, exact},
@@ -344,6 +349,10 @@ get_info(#{} = Db) ->
StatsPrefix = erlfdb_tuple:pack({?DB_STATS}, DbPrefix),
MetaFuture = erlfdb:get_range_startswith(Tx, StatsPrefix),
+ {DbPrefix, ChangesFuture, MetaFuture}.
+
+
+get_info_wait({DbPrefix, ChangesFuture, MetaFuture}) ->
RawSeq = case erlfdb:wait(ChangesFuture) of
[] ->
vs_to_seq(fabric2_util:seq_zero_vs());