You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/02/10 22:39:34 UTC
[couchdb] branch prototype/fdb-layer-get-dbs-info updated (1a7a33e
-> 9719a75)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 1a7a33e fixup: db size tests
new b728bce fixup: add size tracking
new 9719a75 fixup: add size tests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/fabric/src/fabric2_fdb.erl | 17 +---
src/fabric/src/fabric2_util.erl | 33 ++++++-
src/fabric/test/fabric2_doc_size_tests.erl | 138 +++++++++++++++++++++++++++++
3 files changed, 173 insertions(+), 15 deletions(-)
create mode 100644 src/fabric/test/fabric2_doc_size_tests.erl
[couchdb] 02/02: fixup: add size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9719a75a394e6556cebaa23290c4e2c4a6b34645
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 16:41:42 2020 -0600
fixup: add size tests
---
src/fabric/test/fabric2_doc_size_tests.erl | 138 +++++++++++++++++++++++++++++
1 file changed, 138 insertions(+)
diff --git a/src/fabric/test/fabric2_doc_size_tests.erl b/src/fabric/test/fabric2_doc_size_tests.erl
new file mode 100644
index 0000000..4667687
--- /dev/null
+++ b/src/fabric/test/fabric2_doc_size_tests.erl
@@ -0,0 +1,138 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_doc_size_tests).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+% Doc body size calculations
+% ID: size(Doc#doc.id)
+% Rev: size(erlfdb_tuple:encode(Start)) + size(Rev) % where Rev is usually 16
+% Deleted: 1 % (binary value is one byte)
+% Body: couch_ejson_size:external_size(Body) % Where empty is {} which is 2)
+
+
+-define(DOC_IDS, [
+ {0, <<>>},
+ {1, <<"a">>},
+ {3, <<"foo">>},
+ {6, <<"foobar">>},
+ {32, <<"af196ae095631b020eedf8f69303e336">>},
+ {64, <<"af196ae095631b020eedf8f69303e336af196ae095631b020eedf8f69303e336">>}
+]).
+
+-define(REV_STARTS, [
+ {1, 0},
+ {2, 1},
+ {2, 255},
+ {3, 256},
+ {3, 65535},
+ {4, 65536},
+ {4, 16777215},
+ {5, 16777216},
+ {5, 4294967295},
+ {6, 4294967296},
+ {6, 1099511627775},
+ {7, 1099511627776},
+ {7, 281474976710655},
+ {8, 281474976710656},
+ {8, 72057594037927935},
+ {9, 72057594037927936},
+ {9, 18446744073709551615},
+
+ % The jump from 9 to 11 bytes is because when we
+ % spill over into the bigint range of 9-255
+ % bytes we have an extra byte that encodes the
+ % length of the bigint.
+ {11, 18446744073709551616}
+]).
+
+-define(REVS, [
+ {0, <<>>},
+ {8, <<0, 1, 2, 3, 4, 5, 6, 7>>},
+ {16, <<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5>>}
+]).
+
+-define(DELETED, [
+ {1, true},
+ {1, false}
+]).
+
+-define(BODIES, [
+ {2, {[]}},
+ {13, {[{<<"foo">>, <<"bar">>}]}},
+ {28, {[{<<"b">>, <<"a">>}, {<<"c">>, [true, null, []]}]}}
+]).
+
+
+empty_doc_test() ->
+ ?assertEqual(4, fabric2_util:rev_size(#doc{})).
+
+
+docid_size_test() ->
+ lists:foreach(fun({Size, DocId}) ->
+ ?assertEqual(4 + Size, fabric2_util:rev_size(#doc{id = DocId}))
+ end, ?DOC_IDS).
+
+
+rev_size_test() ->
+ lists:foreach(fun({StartSize, Start}) ->
+ lists:foreach(fun({RevSize, Rev}) ->
+ Doc = #doc{
+ revs = {Start, [Rev]}
+ },
+ ?assertEqual(3 + StartSize + RevSize, fabric2_util:rev_size(Doc))
+ end, ?REVS)
+ end, ?REV_STARTS).
+
+
+deleted_size_test() ->
+ lists:foreach(fun({Size, Deleted}) ->
+ ?assertEqual(3 + Size, fabric2_util:rev_size(#doc{deleted = Deleted}))
+ end, ?DELETED).
+
+
+body_size_test() ->
+ lists:foreach(fun({Size, Body}) ->
+ ?assertEqual(2 + Size, fabric2_util:rev_size(#doc{body = Body}))
+ end, ?BODIES).
+
+
+combinatorics_test() ->
+ lists:foreach(fun({DocIdSize, DocId}) ->
+ lists:foreach(fun({RevStartSize, RevStart}) ->
+ lists:foreach(fun({RevSize, Rev}) ->
+ lists:foreach(fun({DeletedSize, Deleted}) ->
+ lists:foreach(fun({BodySize, Body}) ->
+ Doc = #doc{
+ id = DocId,
+ revs = {RevStart, [Rev]},
+ deleted = Deleted,
+ body = Body
+ },
+ Expect = lists:sum([
+ DocIdSize,
+ RevStartSize,
+ RevSize,
+ DeletedSize,
+ BodySize
+ ]),
+ ?assertEqual(Expect, fabric2_util:rev_size(Doc))
+ end, ?BODIES)
+ end, ?DELETED)
+ end, ?REVS)
+ end, ?REV_STARTS)
+ end, ?DOC_IDS).
+
[couchdb] 01/02: fixup: add size tracking
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b728bceacf5c46df1fe294f6d54fb8752154db1b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 16:41:33 2020 -0600
fixup: add size tracking
---
src/fabric/src/fabric2_fdb.erl | 17 +++--------------
src/fabric/src/fabric2_util.erl | 33 ++++++++++++++++++++++++++++++++-
2 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 8d8616d..d5be6d7 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -1317,24 +1317,13 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{{K, Chunk}, ChunkId + 1}
end, 0, chunkify_binary(BVal)),
- % Calculate size
- TotalSize = case Doc#doc.deleted of
- true ->
- 0;
- false ->
- lists:sum([
- size(Id),
- size(StoreRev),
- couch_ejson_size:encoded_size(Body)
- ])
- end,
-
- RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
+ NewSize = fabric2_util:ldoc_size(Doc),
+ RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, NewSize}),
% Prefix our tuple encoding to make upgrades easier
Value = <<255, RawValue/binary>>,
- {Key, Value, TotalSize, Rows}.
+ {Key, Value, NewSize, Rows}.
fdb_to_local_doc(_Db, _DocId, not_found, []) ->
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 0f75390..766441c 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -18,6 +18,7 @@
revinfo_to_path/1,
sort_revinfos/1,
rev_size/1,
+ ldoc_size/1,
seq_zero_vs/0,
seq_max_vs/0,
@@ -82,11 +83,16 @@ rev_sort_key(#{} = RevInfo) ->
rev_size(#doc{} = Doc) ->
#doc{
id = Id,
- revs = {Start, [Rev | _]},
+ revs = Revs,
body = Body,
atts = Atts
} = Doc,
+ {Start, Rev} = case Revs of
+ {0, []} -> {0, <<>>};
+ {N, [RevId | _]} -> {N, RevId}
+ end,
+
lists:sum([
size(Id),
size(erlfdb_tuple:pack({Start})),
@@ -99,6 +105,31 @@ rev_size(#doc{} = Doc) ->
]).
+ldoc_size(#doc{id = <<"_local/">>} = Doc) ->
+ #doc{
+ id = Id,
+ revs = {0, [Rev]},
+ deleted = Deleted,
+ body = Body
+ } = Doc,
+
+ StoreRev = case Rev of
+ _ when is_integer(Rev) -> integer_to_binary(Rev);
+ _ when is_binary(Rev) -> Rev
+ end,
+
+ case Deleted of
+ true ->
+ 0;
+ false ->
+ lists:sum([
+ size(Id),
+ size(StoreRev),
+ couch_ejson_size:encoded_size(Body)
+ ])
+ end.
+
+
seq_zero_vs() ->
{versionstamp, 0, 0, 0}.