You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/02/10 18:40:58 UTC
[couchdb] branch prototype/fdb-layer-get-dbs-info updated (001458a
-> 1a7a33e)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 001458a Add database size tests
discard 9170252 Track the size of data stored in a database
discard da5c356 Support `GET /_dbs_info` endpoint
discard 628a041 Implement `fabric2_db:list_dbs_info/1,2,3`
discard 35e7019 Implement async API for `fabric2_fdb:get_info/1`
discard 3969839 Track a database level view size rollup
new 8918e28 Improve transaction name setting when tracing FDB transactions
new 7507b68 Add `external` tag to opentrace events
new f187627 Change map indexes to be stored in one row
new 5b1d506 Bump ioq to 2.1.3
new a7068fa Delete unused ets table creation
new 24f5349 reserve search namespace
new 3565e52 Support jaeger http reporter
new c83b48c Track a database level view size rollup
new bde305d Implement async API for `fabric2_fdb:get_info/1`
new b2c6938 Implement `fabric2_db:list_dbs_info/1,2,3`
new 916a85f Support `GET /_dbs_info` endpoint
new 87e0795 Track the size of data stored in a database
new 7d95d89 Add database size tests
new db2e8c4 fixup: add db size tracking
new bf8f84b Fix doc attachment tests
new 1a7a33e fixup: db size tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (001458a)
\
N -- N -- N refs/heads/prototype/fdb-layer-get-dbs-info (1a7a33e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rebar.config.script | 6 +-
rel/overlay/etc/default.ini | 12 +-
src/chttpd/src/chttpd.erl | 12 +-
.../src/couch_expiring_cache_server.erl | 1 -
src/couch_views/include/couch_views.hrl | 3 -
src/couch_views/src/couch_views_fdb.erl | 126 +++------------------
src/couch_views/src/couch_views_reader.erl | 2 +-
src/ctrace/README.md | 21 +++-
src/ctrace/src/ctrace_config.erl | 38 +++++--
src/fabric/include/fabric2.hrl | 1 +
src/fabric/src/fabric2_db.erl | 7 +-
src/fabric/src/fabric2_fdb.erl | 70 +++++++-----
src/fabric/src/fabric2_util.erl | 21 ++++
src/fabric/test/fabric2_db_size_tests.erl | 100 ++++++++++------
src/fabric/test/fabric2_doc_att_tests.erl | 4 +-
15 files changed, 221 insertions(+), 203 deletions(-)
[couchdb] 14/16: fixup: add db size tracking
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit db2e8c4db21ab3d5a4f5af4b5db044833edd2f48
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:06 2020 -0600
fixup: add db size tracking
---
src/fabric/src/fabric2_db.erl | 7 +++--
src/fabric/src/fabric2_fdb.erl | 63 +++++++++++++++++++++++------------------
src/fabric/src/fabric2_util.erl | 21 ++++++++++++++
3 files changed, 61 insertions(+), 30 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 26aad75..4528194 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1417,13 +1417,14 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
NewRevInfo = #{
winner => undefined,
+ exists => false,
deleted => NewDeleted,
rev_id => {NewRevPos, NewRev},
rev_path => NewRevPath,
sequence => undefined,
branch_count => undefined,
att_hash => fabric2_util:hash_atts(Atts),
- rev_size => null
+ rev_size => fabric2_util:rev_size(Doc4)
},
% Gather the list of possible winnig revisions
@@ -1474,6 +1475,7 @@ update_doc_replicated(Db, Doc0, _Options) ->
DocRevInfo0 = #{
winner => undefined,
+ exists => false,
deleted => Deleted,
rev_id => {RevPos, Rev},
rev_path => RevPath,
@@ -1520,7 +1522,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
Doc2 = prep_and_validate(Db, Doc1, PrevRevInfo),
Doc3 = flush_doc_atts(Db, Doc2),
DocRevInfo2 = DocRevInfo1#{
- atts_hash => fabric2_util:hash_atts(Doc3#doc.atts)
+ atts_hash => fabric2_util:hash_atts(Doc3#doc.atts),
+ rev_size => fabric2_util:rev_size(Doc3)
},
% Possible winners are the previous winner and
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index f447e93..8d8616d 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -632,7 +632,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Doc body
- {ok, RevSize} = write_doc_body(Db, Doc),
+ ok = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -663,8 +663,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Revision tree
NewWinner = NewWinner0#{
- winner := true,
- rev_size := RevSize
+ winner := true
},
NewRevId = maps:get(rev_id, NewWinner),
@@ -767,13 +766,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
end,
% Update database size
- SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
- Acc + case maps:get(rev_size, RI, null) of
- null -> 0;
- Size -> Size
- end
- end, 0, ToRemove),
- incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+ AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
+ RemSize = sum_rem_rev_sizes(ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, AddSize - RemSize),
ok.
@@ -1091,11 +1086,10 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
- {Rows, RevSize} = doc_to_fdb(Db, Doc),
+ Rows = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, Rows),
- {ok, RevSize}.
+ end, Rows).
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1208,6 +1202,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
{_RevFormat, Sequence, BranchCount, RevPath, AttHash, RevSize} = Val,
#{
winner => true,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1222,6 +1217,7 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
{_RevFormat, RevPath, AttHash, RevSize} = Val,
#{
winner => false,
+ exists => true,
deleted => not NotDeleted,
rev_id => {RevPos, Rev},
rev_path => tuple_to_list(RevPath),
@@ -1240,11 +1236,11 @@ fdb_to_revinfo(Key, {0, RPath}) ->
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, Seq, BCount, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val);
fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
- Val = {?CURR_REV_FORMAT, RPath, AttHash, null},
+ Val = {?CURR_REV_FORMAT, RPath, AttHash, 0},
fdb_to_revinfo(Key, Val).
@@ -1271,19 +1267,7 @@ doc_to_fdb(Db, #doc{} = Doc) ->
{{Key, Chunk}, ChunkId + 1}
end, 0, Chunks),
- % Calculate the size of this revision
- TotalSize = lists:sum([
- size(Id),
- size(erlfdb_tuple:pack({Start})),
- size(Rev),
- 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
- couch_ejson_size:encoded_size(Body),
- lists:foldl(fun(Att, Acc) ->
- couch_att:external_size(Att) + Acc
- end, 0, Atts)
- ]),
-
- {Rows, TotalSize}.
+ Rows.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1388,6 +1372,29 @@ fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
fdb_to_local_doc(Db, DocId, Rev, Rows).
+sum_add_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := Exists,
+ rev_size := Size
+ } = RI,
+ case Exists of
+ true -> Acc;
+ false -> Size + Acc
+ end
+ end, 0, RevInfos).
+
+
+sum_rem_rev_sizes(RevInfos) ->
+ lists:foldl(fun(RI, Acc) ->
+ #{
+ exists := true,
+ rev_size := Size
+ } = RI,
+ Size + Acc
+ end, 0, RevInfos).
+
+
chunkify_binary(Data) ->
case Data of
<<>> ->
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index 4e2e2d7..0f75390 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -17,6 +17,7 @@
revinfo_to_revs/1,
revinfo_to_path/1,
sort_revinfos/1,
+ rev_size/1,
seq_zero_vs/0,
seq_max_vs/0,
@@ -78,6 +79,26 @@ rev_sort_key(#{} = RevInfo) ->
{not Deleted, RevPos, Rev}.
+rev_size(#doc{} = Doc) ->
+ #doc{
+ id = Id,
+ revs = {Start, [Rev | _]},
+ body = Body,
+ atts = Atts
+ } = Doc,
+
+ lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]).
+
+
seq_zero_vs() ->
{versionstamp, 0, 0, 0}.
[couchdb] 04/16: Bump ioq to 2.1.3
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5b1d506f44ecd18e4404f409f5a4d6ddac7d6dff
Author: Eric Avdey <ei...@eiri.ca>
AuthorDate: Tue Jan 28 09:43:20 2020 -0400
Bump ioq to 2.1.3
---
rebar.config.script | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rebar.config.script b/rebar.config.script
index 29c3fde..2fde9c5 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -107,7 +107,7 @@ DepDescs = [
{ets_lru, "ets-lru", {tag, "1.0.0"}},
{khash, "khash", {tag, "1.0.1"}},
{snappy, "snappy", {tag, "CouchDB-1.0.4"}},
-{ioq, "ioq", {tag, "2.1.2"}},
+{ioq, "ioq", {tag, "2.1.3"}},
{hqueue, "hqueue", {tag, "1.0.1"}},
{smoosh, "smoosh", {tag, "1.0.1"}},
{ken, "ken", {tag, "1.0.3"}},
[couchdb] 11/16: Support `GET /_dbs_info` endpoint
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 916a85f8c985ef63dd3c8e2e169be23c36513572
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 13:44:35 2019 -0600
Support `GET /_dbs_info` endpoint
Previously only `POST` with a list of keys was supported. The new `GET`
support just dumps all database info blobs in a single ordered response.
---
src/chttpd/src/chttpd_misc.erl | 49 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 186ec9f..b181f3c 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -157,6 +157,37 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
{ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
{ok, Acc#vacc{resp=Resp1}}.
+handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
+ ok = chttpd:verify_is_server_admin(Req),
+
+ #mrargs{
+ start_key = StartKey,
+ end_key = EndKey,
+ direction = Dir,
+ limit = Limit,
+ skip = Skip
+ } = couch_mrview_http:parse_params(Req, undefined),
+
+ Options = [
+ {start_key, StartKey},
+ {end_key, EndKey},
+ {dir, Dir},
+ {limit, Limit},
+ {skip, Skip}
+ ],
+
+ % TODO: Figure out if we can't calculate a valid
+ % ETag for this request. \xFFmetadataVersion won't
+ % work as we don't bump versions on size changes
+
+ {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, []),
+ Callback = fun dbs_info_callback/2,
+ Acc = #vacc{req = Req, resp = Resp},
+ {ok, Resp} = fabric2_db:list_dbs_info(Callback, Acc, Options),
+ case is_record(Resp, vacc) of
+ true -> {ok, Resp#vacc.resp};
+ _ -> {ok, Resp}
+ end;
handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
chttpd:validate_ctype(Req, "application/json"),
Props = chttpd:json_body_obj(Req),
@@ -188,7 +219,23 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
send_chunk(Resp, "]"),
chttpd:end_json_response(Resp);
handle_dbs_info_req(Req) ->
- send_method_not_allowed(Req, "POST").
+ send_method_not_allowed(Req, "GET,HEAD,POST").
+
+dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
+ {ok, Acc#vacc{resp = Resp1}};
+dbs_info_callback({row, Props}, #vacc{resp = Resp0} = Acc) ->
+ Prepend = couch_mrview_http:prepend_val(Acc),
+ Chunk = [Prepend, ?JSON_ENCODE({Props})],
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+ {ok, Acc#vacc{prepend = ",", resp = Resp1}};
+dbs_info_callback(complete, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, Acc#vacc{resp = Resp2}};
+dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
+ {ok, Acc#vacc{resp = Resp1}}.
handle_task_status_req(#httpd{method='GET'}=Req) ->
ok = chttpd:verify_is_server_admin(Req),
[couchdb] 15/16: Fix doc attachment tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bf8f84b7275b047e16e00d3d868fc9a9b97d020b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:22 2020 -0600
Fix doc attachment tests
---
src/fabric/test/fabric2_doc_att_tests.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/fabric/test/fabric2_doc_att_tests.erl b/src/fabric/test/fabric2_doc_att_tests.erl
index 331e1a4..ac531e9 100644
--- a/src/fabric/test/fabric2_doc_att_tests.erl
+++ b/src/fabric/test/fabric2_doc_att_tests.erl
@@ -175,9 +175,9 @@ large_att({Db, _}) ->
AttData = iolist_to_binary([
<<"foobar">> || _ <- lists:seq(1, 60000)
]),
- Att1 = mk_att("long.txt", AttData),
+ Att1 = mk_att(<<"long.txt">>, AttData),
{ok, _} = create_doc(Db, DocId, [Att1]),
- ?assertEqual(#{"long.txt" => AttData}, read_atts(Db, DocId)),
+ ?assertEqual(#{<<"long.txt">> => AttData}, read_atts(Db, DocId)),
{ok, Doc} = fabric2_db:open_doc(Db, DocId),
#doc{atts = [Att2]} = Doc,
[couchdb] 06/16: reserve search namespace
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 24f534992e97b34c4d2570f4eaf7ac80882f930f
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Jan 29 20:41:07 2020 +0000
reserve search namespace
---
src/fabric/include/fabric2.hrl | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index b4dd084..828a51b 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -38,6 +38,7 @@
-define(DB_VIEWS, 24).
-define(DB_LOCAL_DOC_BODIES, 25).
-define(DB_ATT_NAMES, 26).
+-define(DB_SEARCH, 27).
% Versions
[couchdb] 16/16: fixup: db size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1a7a33e2a90a2e0ff59e8aafbb55e5c68dd328d6
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Feb 10 12:42:50 2020 -0600
fixup: db size tests
---
src/fabric/test/fabric2_db_size_tests.erl | 100 ++++++++++++++++++++----------
1 file changed, 68 insertions(+), 32 deletions(-)
diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
index fc95e0e..f9d514d 100644
--- a/src/fabric/test/fabric2_db_size_tests.erl
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -19,6 +19,24 @@
-include("fabric2_test.hrl").
+-define(DIFF(Db, Change, Fun), begin
+ ((fun() ->
+ __Before = db_size(Db),
+ __Result = Fun(),
+ __After = db_size(Db),
+ ?debugFmt("~p - ~p == ~p ?= ~p", [
+ __After,
+ __Before,
+ __After - __Before,
+ Change
+ ]),
+ ?assertEqual(Change, __After - __Before),
+ __Result
+ end)())
+ end
+).
+
+
db_size_test_() ->
{
"Test document CRUD operations",
@@ -32,6 +50,7 @@ db_size_test_() ->
?TDEF(edit_doc),
?TDEF(del_doc),
?TDEF(conflicted_doc),
+ ?TDEF(del_winner),
?TDEF(del_conflict)
])
}
@@ -54,65 +73,96 @@ empty_size({Db, _}) ->
new_doc({Db, _}) ->
- increases(Db, fun() ->
+ % UUID doc id: 32
+ % Revision: 2 + 16
+ % Deleted: 1
+ % Body: {} = 2
+ ?DIFF(Db, 53, fun() ->
create_doc(Db)
end).
edit_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 53, fun() ->
create_doc(Db, DocId)
end),
- {ok, RevId2} = increases(Db, fun() ->
+ % {} -> {"foo":"bar"} = 13 - 2
+ {ok, RevId2} = ?DIFF(Db, 11, fun() ->
update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- decreases(Db, fun() ->
+ ?DIFF(Db, -11, fun() ->
update_doc(Db, DocId, RevId2)
end).
del_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId} = increases(Db, fun() ->
+ {ok, RevId} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
% The change here is -11 becuase we're going from
% {"foo":"bar"} == 13 bytes to {} == 2 bytes.
% I.e., 2 - 13 == -11
- diff(Db, fun() ->
+ ?DIFF(Db, -11, fun() ->
delete_doc(Db, DocId, RevId)
- end, -11).
+ end).
+% need to check both new conflict is new winner
+% and that new conflict is not a winner and that
+% the sizes don't interfere which should be doable
+% with different sized bodies.
+
conflicted_doc({Db, _}) ->
DocId = fabric2_util:uuid(),
- Before = db_size(Db),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ ?DIFF(Db, 64, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end).
+
+
+del_winner({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
- Between = db_size(Db),
- increases(Db, fun() ->
+ {ok, RevId2} = ?DIFF(Db, 64, fun() ->
create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- After = db_size(Db),
- ?assertEqual(After - Between, Between - Before).
+ [_ConflictRev, WinnerRev] = lists:sort([RevId1, RevId2]),
+ ?DIFF(Db, -11, fun() ->
+ {ok, _RevId3} = delete_doc(Db, DocId, WinnerRev),
+ ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
+ end).
del_conflict({Db, _}) ->
DocId = fabric2_util:uuid(),
- {ok, RevId1} = increases(Db, fun() ->
+ {ok, RevId1} = ?DIFF(Db, 64, fun() ->
create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
end),
- {ok, RevId2} = increases(Db, fun() ->
+ {ok, RevId2} = ?DIFF(Db, 64, fun() ->
create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
end),
- decreases(Db, fun() ->
- {ok, RevId3} = delete_doc(Db, DocId, RevId2),
- ?debugFmt("~p ~p ~p", [RevId1, RevId2, RevId3])
+ [ConflictRev, _WinnerRev] = lists:sort([RevId1, RevId2]),
+ ?DIFF(Db, -11, fun() ->
+ {ok, _RevId3} = delete_doc(Db, DocId, ConflictRev),
+ ?debugFmt("~n~w~n~w~n~w~n", [RevId1, RevId2, _RevId3])
end).
+% replicate with attachment
+% replicate removing attachment
+% replicate reusing attachment
+% replicate adding attachment with stub
+% for each, replicate to winner vs non-winner
+% for each, replicate extending winner, vs extending conflict vs new branch
+
+
+
create_doc(Db) ->
create_doc(Db, fabric2_util:uuid()).
@@ -175,20 +225,6 @@ delete_doc(Db, DocId, {Pos, Rev}, Body) ->
fabric2_db:update_doc(Db, Doc).
-constant(Db, Fun) -> check(Db, Fun, fun erlang:'=='/2).
-increases(Db, Fun) -> check(Db, Fun, fun erlang:'>'/2).
-decreases(Db, Fun) -> check(Db, Fun, fun erlang:'<'/2).
-diff(Db, Fun, Change) -> check(Db, Fun, fun(A, B) -> (A - B) == Change end).
-
-check(Db, Fun, Cmp) ->
- Before = db_size(Db),
- Result = Fun(),
- After = db_size(Db),
- ?debugFmt("~p :: ~p ~p", [erlang:fun_info(Cmp), After, Before]),
- ?assert(Cmp(After, Before)),
- Result.
-
-
db_size(Info) when is_list(Info) ->
{sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
{<<"external">>, External} = lists:keyfind(<<"external">>, 1, Sizes),
[couchdb] 02/16: Add `external` tag to opentrace events
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7507b686b109c5e615dea3921b69f8827659891f
Author: ILYA Khlopotov <ii...@apache.org>
AuthorDate: Tue Jan 14 02:41:50 2020 -0800
Add `external` tag to opentrace events
This PR adds an ability to selectively enable opentracing for HTTP requests
with X-B3-... headers. This is helpful in following cases:
- tracing all requests with X-B3-... headers
`all = (#{external := E}) when E == true -> true`
- tracing all requests to specific database with X-B3-... headers
```
all = (#{external := E, 'db.name' := Db})
when E == true andalso Db == <<"foo">> -> true
```
- tracing requests to specific endpoint with X-B3-... headers
```
db.design.view.read = (#{external := E, 'design.id' := Name})
when E == true andalso Name == <<"bar">> -> true
```
I want to remind that we support following X-B3-... headers:
- X-B3-TraceId
- X-B3-SpanId
- X-B3-ParentSpanId
- B3 which is in the following format
<TraceId>-<SpanId>-<1 | 0>-<ParentSpanId>
---
src/chttpd/src/chttpd.erl | 12 +++++++-----
src/ctrace/README.md | 3 +++
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index c4bfa60..d5e7314 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1250,6 +1250,7 @@ start_span(Req) ->
[] -> <<"">>;
[_ | _] -> filename:join(PathParts)
end,
+ {IsExternalSpan, RootOptions} = root_span_options(MochiReq),
Tags = maps:merge(#{
peer => Peer,
'http.method' => Method,
@@ -1257,21 +1258,22 @@ start_span(Req) ->
'http.url' => MochiReq:get(raw_path),
path_parts => Path,
'span.kind' => <<"server">>,
- component => <<"couchdb.chttpd">>
+ component => <<"couchdb.chttpd">>,
+ external => IsExternalSpan
}, ExtraTags),
ctrace:start_span(OperationName, [
{tags, Tags},
{time, Begin}
- ] ++ maybe_root_span(MochiReq)).
+ ] ++ RootOptions).
-maybe_root_span(MochiReq) ->
+root_span_options(MochiReq) ->
case get_trace_headers(MochiReq) of
[undefined, _, _] ->
- [];
+ {false, []};
[TraceId, SpanId, ParentSpanId] ->
Span = ctrace:external_span(TraceId, SpanId, ParentSpanId),
- [{root, Span}]
+ {true, [{root, Span}]}
end.
parse_trace_id(undefined) ->
diff --git a/src/ctrace/README.md b/src/ctrace/README.md
index 6e40b43..3172f26 100644
--- a/src/ctrace/README.md
+++ b/src/ctrace/README.md
@@ -146,7 +146,10 @@ and logged.
```ini
[tracing.filters]
+; trace all events
; all = (#{}) -> true
+; trace all events with X-B3-... headers
+; all = (#{external := External}) when External == true -> true
; database-info.read = (#{'http.method' := Method}) when Method == 'GET' -> true
; view.build = (#{'view.name' := Name}) when Name == "foo" -> 0.25
```
[couchdb] 08/16: Track a database level view size rollup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c83b48c759ffedb8a9154ceb8a2dc07b8150618a
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 10:24:36 2019 -0600
Track a database level view size rollup
This way we can expose the total view size for a database in the dbinfo
JSON blob.
---
src/couch_views/src/couch_views_fdb.erl | 17 ++++++++++++++--
src/fabric/src/fabric2_fdb.erl | 36 ++++++++++++++++-----------------
2 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 98cff46..5edaa3a 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -272,8 +272,16 @@ update_kv_size(TxDb, Sig, ViewId, Increment) ->
tx := Tx,
db_prefix := DbPrefix
} = TxDb,
- Key = kv_size_key(DbPrefix, Sig, ViewId),
- erlfdb:add(Tx, Key, Increment).
+
+ % Track a view specific size for calls to
+ % GET /dbname/_design/doc/_info`
+ IdxKey = kv_size_key(DbPrefix, Sig, ViewId),
+ erlfdb:add(Tx, IdxKey, Increment),
+
+ % Track a database level rollup for calls to
+ % GET /dbname
+ DbKey = db_kv_size_key(DbPrefix),
+ erlfdb:add(Tx, DbKey, Increment).
seq_key(DbPrefix, Sig) ->
@@ -291,6 +299,11 @@ kv_size_key(DbPrefix, Sig, ViewId) ->
erlfdb_tuple:pack(Key, DbPrefix).
+db_kv_size_key(DbPrefix) ->
+ Key = {?DB_STATS, <<"sizes">>, <<"views">>},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 6abe1f6..8bfbb74 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -174,11 +174,16 @@ create(#{} = Db0, Options) ->
{?DB_STATS, <<"doc_del_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_design_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_local_count">>, ?uint2bin(0)},
- {?DB_STATS, <<"size">>, ?uint2bin(2)}
+ {?DB_STATS, <<"sizes">>, <<"external">>, ?uint2bin(2)},
+ {?DB_STATS, <<"sizes">>, <<"views">>, ?uint2bin(0)}
],
- lists:foreach(fun({P, K, V}) ->
- Key = erlfdb_tuple:pack({P, K}, DbPrefix),
- erlfdb:set(Tx, Key, V)
+ lists:foreach(fun
+ ({P, K, V}) ->
+ Key = erlfdb_tuple:pack({P, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V);
+ ({P, S, K, V}) ->
+ Key = erlfdb_tuple:pack({P, S, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V)
end, Defaults),
UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}),
@@ -348,26 +353,21 @@ get_info(#{} = Db) ->
end,
CProp = {update_seq, RawSeq},
- MProps = lists:flatmap(fun({K, V}) ->
+ MProps = lists:foldl(fun({K, V}, Acc) ->
case erlfdb_tuple:unpack(K, DbPrefix) of
{?DB_STATS, <<"doc_count">>} ->
- [{doc_count, ?bin2uint(V)}];
+ [{doc_count, ?bin2uint(V)} | Acc];
{?DB_STATS, <<"doc_del_count">>} ->
- [{doc_del_count, ?bin2uint(V)}];
- {?DB_STATS, <<"size">>} ->
+ [{doc_del_count, ?bin2uint(V)} | Acc];
+ {?DB_STATS, <<"sizes">>, Name} ->
Val = ?bin2uint(V),
- [
- {other, {[{data_size, Val}]}},
- {sizes, {[
- {active, 0},
- {external, Val},
- {file, 0}
- ]}}
- ];
+ {_, {Sizes}} = lists:keyfind(sizes, 1, Acc),
+ NewSizes = lists:keystore(Name, 1, Sizes, {Name, Val}),
+ lists:keystore(sizes, 1, Acc, {sizes, {NewSizes}});
{?DB_STATS, _} ->
- []
+ Acc
end
- end, erlfdb:wait(MetaFuture)),
+ end, [{sizes, {[]}}], erlfdb:wait(MetaFuture)),
[CProp | MProps].
[couchdb] 09/16: Implement async API for `fabric2_fdb:get_info/1`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bde305dead63bc920618b7929832a01548c4a922
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:44:34 2019 -0600
Implement async API for `fabric2_fdb:get_info/1`
---
src/fabric/src/fabric2_fdb.erl | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 8bfbb74..0e7cba8 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -29,6 +29,8 @@
list_dbs/4,
get_info/1,
+ get_info_future/2,
+ get_info_wait/1,
set_config/3,
get_stat/2,
@@ -333,7 +335,10 @@ get_info(#{} = Db) ->
tx := Tx,
db_prefix := DbPrefix
} = ensure_current(Db),
+ get_info_wait(get_info_future(Tx, DbPrefix)).
+
+get_info_future(Tx, DbPrefix) ->
{CStart, CEnd} = erlfdb_tuple:range({?DB_CHANGES}, DbPrefix),
ChangesFuture = erlfdb:get_range(Tx, CStart, CEnd, [
{streaming_mode, exact},
@@ -344,6 +349,10 @@ get_info(#{} = Db) ->
StatsPrefix = erlfdb_tuple:pack({?DB_STATS}, DbPrefix),
MetaFuture = erlfdb:get_range_startswith(Tx, StatsPrefix),
+ {DbPrefix, ChangesFuture, MetaFuture}.
+
+
+get_info_wait({DbPrefix, ChangesFuture, MetaFuture}) ->
RawSeq = case erlfdb:wait(ChangesFuture) of
[] ->
vs_to_seq(fabric2_util:seq_zero_vs());
[couchdb] 13/16: Add database size tests
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7d95d89ea5f8b59dd611e3eae59ce038b47c43ac
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 6 14:59:30 2019 -0600
Add database size tests
---
src/fabric/test/fabric2_db_size_tests.erl | 198 ++++++++++++++++++++++++++++++
1 file changed, 198 insertions(+)
diff --git a/src/fabric/test/fabric2_db_size_tests.erl b/src/fabric/test/fabric2_db_size_tests.erl
new file mode 100644
index 0000000..fc95e0e
--- /dev/null
+++ b/src/fabric/test/fabric2_db_size_tests.erl
@@ -0,0 +1,198 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric2_db_size_tests).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("fabric2_test.hrl").
+
+
+db_size_test_() ->
+ {
+ "Test document CRUD operations",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ with([
+ ?TDEF(empty_size),
+ ?TDEF(new_doc),
+ ?TDEF(edit_doc),
+ ?TDEF(del_doc),
+ ?TDEF(conflicted_doc),
+ ?TDEF(del_conflict)
+ ])
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([fabric]),
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ {Db, Ctx}.
+
+
+cleanup({Db, Ctx}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []),
+ test_util:stop_couch(Ctx).
+
+
+empty_size({Db, _}) ->
+ ?assertEqual(2, db_size(Db)).
+
+
+new_doc({Db, _}) ->
+ increases(Db, fun() ->
+ create_doc(Db)
+ end).
+
+
+edit_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId)
+ end),
+ {ok, RevId2} = increases(Db, fun() ->
+ update_doc(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ decreases(Db, fun() ->
+ update_doc(Db, DocId, RevId2)
+ end).
+
+
+del_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ % The change here is -11 becuase we're going from
+ % {"foo":"bar"} == 13 bytes to {} == 2 bytes.
+ % I.e., 2 - 13 == -11
+ diff(Db, fun() ->
+ delete_doc(Db, DocId, RevId)
+ end, -11).
+
+
+conflicted_doc({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ Before = db_size(Db),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ Between = db_size(Db),
+ increases(Db, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ After = db_size(Db),
+ ?assertEqual(After - Between, Between - Before).
+
+
+del_conflict({Db, _}) ->
+ DocId = fabric2_util:uuid(),
+ {ok, RevId1} = increases(Db, fun() ->
+ create_doc(Db, DocId, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ {ok, RevId2} = increases(Db, fun() ->
+ create_conflict(Db, DocId, RevId1, {[{<<"foo">>, <<"bar">>}]})
+ end),
+ decreases(Db, fun() ->
+ {ok, RevId3} = delete_doc(Db, DocId, RevId2),
+ ?debugFmt("~p ~p ~p", [RevId1, RevId2, RevId3])
+ end).
+
+
+create_doc(Db) ->
+ create_doc(Db, fabric2_util:uuid()).
+
+
+create_doc(Db, DocId) when is_binary(DocId) ->
+ create_doc(Db, DocId, {[]});
+create_doc(Db, {Props} = Body) when is_list(Props) ->
+ create_doc(Db, fabric2_util:uuid(), Body).
+
+
+create_doc(Db, DocId, Body) ->
+ Doc = #doc{
+ id = DocId,
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+create_conflict(Db, DocId, RevId) ->
+ create_conflict(Db, DocId, RevId, {[]}).
+
+
+create_conflict(Db, DocId, RevId, Body) ->
+ {Pos, _} = RevId,
+ % Only keep the first 16 bytes of the UUID
+ % so that we match the normal sized revs
+ <<NewRev:16/binary, _/binary>> = fabric2_util:uuid(),
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [NewRev]},
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc, [replicated_changes]).
+
+
+update_doc(Db, DocId, RevId) ->
+ update_doc(Db, DocId, RevId, {[]}).
+
+
+update_doc(Db, DocId, {Pos, Rev}, Body) ->
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [Rev]},
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+delete_doc(Db, DocId, RevId) ->
+ delete_doc(Db, DocId, RevId, {[]}).
+
+
+delete_doc(Db, DocId, {Pos, Rev}, Body) ->
+ Doc = #doc{
+ id = DocId,
+ revs = {Pos, [Rev]},
+ deleted = true,
+ body = Body
+ },
+ fabric2_db:update_doc(Db, Doc).
+
+
+constant(Db, Fun) -> check(Db, Fun, fun erlang:'=='/2).
+increases(Db, Fun) -> check(Db, Fun, fun erlang:'>'/2).
+decreases(Db, Fun) -> check(Db, Fun, fun erlang:'<'/2).
+diff(Db, Fun, Change) -> check(Db, Fun, fun(A, B) -> (A - B) == Change end).
+
+check(Db, Fun, Cmp) ->
+ Before = db_size(Db),
+ Result = Fun(),
+ After = db_size(Db),
+ ?debugFmt("~p :: ~p ~p", [erlang:fun_info(Cmp), After, Before]),
+ ?assert(Cmp(After, Before)),
+ Result.
+
+
+db_size(Info) when is_list(Info) ->
+ {sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
+ {<<"external">>, External} = lists:keyfind(<<"external">>, 1, Sizes),
+ External;
+db_size(Db) when is_map(Db) ->
+ {ok, Info} = fabric2_db:get_db_info(Db),
+ db_size(Info).
[couchdb] 03/16: Change map indexes to be stored in one row
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f1876277e447b0da8483e067fd077cc4b16f25eb
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Jan 16 15:29:08 2020 +0200
Change map indexes to be stored in one row
Changes map indexes to store the original key and value in a single
FDB row.
---
src/couch_views/include/couch_views.hrl | 3 -
src/couch_views/src/couch_views_fdb.erl | 126 ++++-------------------------
src/couch_views/src/couch_views_reader.erl | 2 +-
3 files changed, 17 insertions(+), 114 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 2e443eb..642431d 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -19,8 +19,5 @@
-define(VIEW_ROW_COUNT, 0).
-define(VIEW_KV_SIZE, 1).
--define(VIEW_ROW_KEY, 0).
--define(VIEW_ROW_VALUE, 1).
-
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 60ce300..98cff46 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -95,31 +95,14 @@ fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
MapIdxPrefix = map_idx_prefix(DbPrefix, Sig, ViewId),
FoldAcc = #{
prefix => MapIdxPrefix,
- sort_key => undefined,
- docid => undefined,
- dupe_id => undefined,
callback => Callback,
acc => Acc0
- },
-
- {Fun, Acc} = case fabric2_util:get_value(dir, Options, fwd) of
- fwd ->
- FwdAcc = FoldAcc#{
- next => key,
- key => undefined
- },
- {fun fold_fwd/2, FwdAcc};
- rev ->
- RevAcc = FoldAcc#{
- next => value,
- value => undefined
- },
- {fun fold_rev/2, RevAcc}
- end,
+ },
+ Fun = fun fold_fwd/2,
#{
acc := Acc1
- } = fabric2_fdb:fold_range(TxDb, MapIdxPrefix, Fun, Acc, Options),
+ } = fabric2_fdb:fold_range(TxDb, MapIdxPrefix, Fun, FoldAcc, Options),
Acc1.
@@ -169,110 +152,34 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
end, lists:zip(ViewIds, Results)).
-% For each row in a map view there are two rows stored in
-% FoundationDB:
+% For each row in a map view we store the the key/value
+% in FoundationDB:
%
-% `(EncodedSortKey, EncodedKey)`
-% `(EncodedSortKey, EncodedValue)`
+% `(EncodedSortKey, (EncodedKey, EncodedValue))`
%
% The difference between `EncodedSortKey` and `EndcodedKey` is
% the use of `couch_util:get_sort_key/1` which turns UTF-8
% strings into binaries that are byte comparable. Given a sort
% key binary we cannot recover the input so to return unmodified
% user data we are forced to store the original.
-%
-% These two fold functions exist so that we can be fairly
-% forceful on our assertions about which rows to see. Since
-% when we're folding forward we'll see the key first. When
-% `descending=true` and we're folding in reverse we'll see
-% the value first.
-fold_fwd({RowKey, EncodedOriginalKey}, #{next := key} = Acc) ->
- #{
- prefix := Prefix
- } = Acc,
-
- {{SortKey, DocId}, DupeId, ?VIEW_ROW_KEY} =
- erlfdb_tuple:unpack(RowKey, Prefix),
- Acc#{
- next := value,
- key := couch_views_encoding:decode(EncodedOriginalKey),
- sort_key := SortKey,
- docid := DocId,
- dupe_id := DupeId
- };
-
-fold_fwd({RowKey, EncodedValue}, #{next := value} = Acc) ->
+fold_fwd({RowKey, PackedKeyValue}, Acc) ->
#{
prefix := Prefix,
- key := Key,
- sort_key := SortKey,
- docid := DocId,
- dupe_id := DupeId,
callback := UserCallback,
acc := UserAcc0
} = Acc,
- % We're asserting there that this row is paired
- % correctly with the previous row by relying on
- % a badmatch if any of these values don't match.
- {{SortKey, DocId}, DupeId, ?VIEW_ROW_VALUE} =
+ {{_SortKey, DocId}, _DupeId} =
erlfdb_tuple:unpack(RowKey, Prefix),
+ {EncodedOriginalKey, EncodedValue} = erlfdb_tuple:unpack(PackedKeyValue),
Value = couch_views_encoding:decode(EncodedValue),
- UserAcc1 = UserCallback(DocId, Key, Value, UserAcc0),
-
- Acc#{
- next := key,
- key := undefined,
- sort_key := undefined,
- docid := undefined,
- dupe_id := undefined,
- acc := UserAcc1
- }.
-
-
-fold_rev({RowKey, EncodedValue}, #{next := value} = Acc) ->
- #{
- prefix := Prefix
- } = Acc,
-
- {{SortKey, DocId}, DupeId, ?VIEW_ROW_VALUE} =
- erlfdb_tuple:unpack(RowKey, Prefix),
- Acc#{
- next := key,
- value := couch_views_encoding:decode(EncodedValue),
- sort_key := SortKey,
- docid := DocId,
- dupe_id := DupeId
- };
-
-fold_rev({RowKey, EncodedOriginalKey}, #{next := key} = Acc) ->
- #{
- prefix := Prefix,
- value := Value,
- sort_key := SortKey,
- docid := DocId,
- dupe_id := DupeId,
- callback := UserCallback,
- acc := UserAcc0
- } = Acc,
-
- % We're asserting there that this row is paired
- % correctly with the previous row by relying on
- % a badmatch if any of these values don't match.
- {{SortKey, DocId}, DupeId, ?VIEW_ROW_KEY} =
- erlfdb_tuple:unpack(RowKey, Prefix),
-
Key = couch_views_encoding:decode(EncodedOriginalKey),
+
UserAcc1 = UserCallback(DocId, Key, Value, UserAcc0),
Acc#{
- next := value,
- value := undefined,
- sort_key := undefined,
- docid := undefined,
- dupe_id := undefined,
acc := UserAcc1
}.
@@ -330,11 +237,10 @@ update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) ->
KVsToAdd = process_rows(NewRows),
MapIdxPrefix = map_idx_prefix(DbPrefix, Sig, ViewId),
- lists:foreach(fun({DupeId, Key1, Key2, Val}) ->
- KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId, ?VIEW_ROW_KEY),
- VK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId, ?VIEW_ROW_VALUE),
- ok = erlfdb:set(Tx, KK, Key2),
- ok = erlfdb:set(Tx, VK, Val)
+ lists:foreach(fun({DupeId, Key1, Key2, EV}) ->
+ KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId),
+ Val = erlfdb_tuple:pack({Key2, EV}),
+ ok = erlfdb:set(Tx, KK, Val)
end, KVsToAdd).
@@ -400,8 +306,8 @@ map_idx_prefix(DbPrefix, Sig, ViewId) ->
erlfdb_tuple:pack(Key, DbPrefix).
-map_idx_key(MapIdxPrefix, MapKey, DupeId, Type) ->
- Key = {MapKey, DupeId, Type},
+map_idx_key(MapIdxPrefix, MapKey, DupeId) ->
+ Key = {MapKey, DupeId},
erlfdb_tuple:pack(Key, MapIdxPrefix).
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 27671fb..76dbed1 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -183,7 +183,7 @@ mrargs_to_fdb_options(Args) ->
[
{dir, Direction},
- {limit, Limit * 2 + Skip * 2},
+ {limit, Limit + Skip},
{streaming_mode, want_all}
] ++ StartKeyOpts ++ EndKeyOpts.
[couchdb] 01/16: Improve transaction name setting when tracing FDB
transactions
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8918e28b19a04b2991a6ce9525351426a38fabee
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Wed Dec 18 13:49:03 2019 -0500
Improve transaction name setting when tracing FDB transactions
Previously the per-request nonce value was set as the transaction name and so
in the trace logs multiple transactions ended up having the same `TransactionID`
which was pretty confusing.
To fix the issue, append a transaction ID to the name. The ID is guaranteed to
be unique for the life of the VM node.
---
src/fabric/src/fabric2_fdb.erl | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 404460e..6abe1f6 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -118,7 +118,10 @@ do_transaction(Fun, LayerPrefix) when is_function(Fun, 1) ->
erlfdb:transactional(Db, fun(Tx) ->
case get(erlfdb_trace) of
Name when is_binary(Name) ->
- erlfdb:set_option(Tx, transaction_logging_enable, Name);
+ UId = erlang:unique_integer([positive]),
+ UIdBin = integer_to_binary(UId, 36),
+ TxId = <<Name/binary, "_", UIdBin/binary>>,
+ erlfdb:set_option(Tx, transaction_logging_enable, TxId);
_ ->
ok
end,
[couchdb] 12/16: Track the size of data stored in a database
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 87e079554e664d13ff47edf692cfb1444c19a40c
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Dec 4 11:38:48 2019 -0600
Track the size of data stored in a database
This tracks the number of bytes that would be required to store the
contents of a database as flat files on disk. Currently the following
items are tracked:
* Doc ids
* Revisions
* Doc body as JSON
* Attachment names
* Attachment type
* Attachment length
* Attachment md5s
* Attachment headers
* Local doc id
* Local doc revision
* Local doc bodies
---
src/couch/src/couch_att.erl | 19 ++++
src/fabric/include/fabric2.hrl | 7 +-
src/fabric/src/fabric2_db.erl | 6 +-
src/fabric/src/fabric2_fdb.erl | 152 ++++++++++++++++++++++++-----
src/fabric/test/fabric2_doc_crud_tests.erl | 5 +-
5 files changed, 159 insertions(+), 30 deletions(-)
diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index 2c33362..90d498c 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -27,6 +27,7 @@
]).
-export([
+ external_size/1,
size_info/1,
to_disk_term/1,
from_disk_term/3
@@ -179,6 +180,24 @@ merge_stubs([], _, Merged) ->
{ok, lists:reverse(Merged)}.
+external_size(Att) ->
+ NameSize = size(fetch(name, Att)),
+ TypeSize = case fetch(type, Att) of
+ undefined -> 0;
+ Type -> size(Type)
+ end,
+ AttSize = fetch(att_len, Att),
+ Md5Size = case fetch(md5, Att) of
+ undefined -> 0;
+ Md5 -> size(Md5)
+ end,
+ HeadersSize = case fetch(headers, Att) of
+ undefined -> 0;
+ Headers -> couch_ejson_size:encoded_size(Headers)
+ end,
+ NameSize + TypeSize + AttSize + Md5Size + HeadersSize.
+
+
size_info([]) ->
{ok, []};
size_info(Atts) ->
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 828a51b..5f2571e 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -45,8 +45,13 @@
% 0 - Initial implementation
% 1 - Added attachment hash
+% 2 - Added size information
--define(CURR_REV_FORMAT, 1).
+-define(CURR_REV_FORMAT, 2).
+
+% 0 - Adding local doc versions
+
+-define(CURR_LDOC_FORMAT, 0).
% Misc constants
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 17c899d..26aad75 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -1422,7 +1422,8 @@ update_doc_interactive(Db, Doc0, Future, _Options) ->
rev_path => NewRevPath,
sequence => undefined,
branch_count => undefined,
- att_hash => fabric2_util:hash_atts(Atts)
+ att_hash => fabric2_util:hash_atts(Atts),
+ rev_size => null
},
% Gather the list of possible winnig revisions
@@ -1478,7 +1479,8 @@ update_doc_replicated(Db, Doc0, _Options) ->
rev_path => RevPath,
sequence => undefined,
branch_count => undefined,
- att_hash => <<>>
+ att_hash => <<>>,
+ rev_size => null
},
AllRevInfos = fabric2_fdb:get_all_revs(Db, DocId),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 99611b0..f447e93 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -36,6 +36,7 @@
get_stat/2,
incr_stat/3,
+ incr_stat/4,
get_all_revs/2,
get_winning_revs/3,
@@ -454,6 +455,19 @@ incr_stat(#{} = Db, StatKey, Increment) when is_integer(Increment) ->
erlfdb:add(Tx, Key, Increment).
+incr_stat(_Db, _Section, _Key, 0) ->
+ ok;
+
+incr_stat(#{} = Db, Section, Key, Increment) when is_integer(Increment) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = ensure_current(Db),
+
+ BinKey = erlfdb_tuple:pack({?DB_STATS, Section, Key}, DbPrefix),
+ erlfdb:add(Tx, BinKey, Increment).
+
+
get_all_revs(#{} = Db, DocId) ->
#{
tx := Tx,
@@ -573,6 +587,15 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) ->
get_local_doc_rev(_Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId, Val) ->
case Val of
+ <<255, RevBin/binary>> ->
+ % Versioned local docs
+ try
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev, _Size} -> Rev
+ end
+ catch _:_ ->
+ erlang:error({invalid_local_doc_rev, DocId, Val})
+ end;
<<131, _/binary>> ->
% Compatibility clause for an older encoding format
try binary_to_term(Val, [safe]) of
@@ -609,7 +632,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Doc body
- ok = write_doc_body(Db, Doc),
+ {ok, RevSize} = write_doc_body(Db, Doc),
% Attachment bookkeeping
@@ -639,7 +662,10 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
% Revision tree
- NewWinner = NewWinner0#{winner := true},
+ NewWinner = NewWinner0#{
+ winner := true,
+ rev_size := RevSize
+ },
NewRevId = maps:get(rev_id, NewWinner),
{WKey, WVal, WinnerVS} = revinfo_to_fdb(Tx, DbPrefix, DocId, NewWinner),
@@ -701,7 +727,7 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
NewSeqVal = erlfdb_tuple:pack({DocId, Deleted, NewRevId}),
erlfdb:set_versionstamped_key(Tx, NewSeqKey, NewSeqVal),
- % And all the rest...
+ % Bump db version on design doc changes
IsDDoc = case Doc#doc.id of
<<?DESIGN_DOC_PREFIX, _/binary>> -> true;
@@ -712,6 +738,8 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
bump_db_version(Db)
end,
+ % Update our document counts
+
case UpdateStatus of
created ->
if not IsDDoc -> ok; true ->
@@ -738,6 +766,15 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ % Update database size
+ SizeIncr = RevSize - lists:foldl(fun(RI, Acc) ->
+ Acc + case maps:get(rev_size, RI, null) of
+ null -> 0;
+ Size -> Size
+ end
+ end, 0, ToRemove),
+ incr_stat(Db, <<"sizes">>, <<"external">>, SizeIncr),
+
ok.
@@ -749,11 +786,18 @@ write_local_doc(#{} = Db0, Doc) ->
Id = Doc#doc.id,
- {LDocKey, LDocVal, Rows} = local_doc_to_fdb(Db, Doc),
+ {LDocKey, LDocVal, NewSize, Rows} = local_doc_to_fdb(Db, Doc),
- WasDeleted = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
- <<_/binary>> -> false;
- not_found -> true
+ {WasDeleted, PrevSize} = case erlfdb:wait(erlfdb:get(Tx, LDocKey)) of
+ <<255, RevBin/binary>> ->
+ case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, _Rev, Size} ->
+ {false, Size}
+ end;
+ <<_/binary>> ->
+ {false, 0};
+ not_found ->
+ {true, 0}
end,
BPrefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, Id}, DbPrefix),
@@ -779,6 +823,8 @@ write_local_doc(#{} = Db0, Doc) ->
ok
end,
+ incr_stat(Db, <<"sizes">>, <<"external">>, NewSize - PrevSize),
+
ok.
@@ -1045,9 +1091,11 @@ write_doc_body(#{} = Db0, #doc{} = Doc) ->
tx := Tx
} = Db = ensure_current(Db0),
+ {Rows, RevSize} = doc_to_fdb(Db, Doc),
lists:foreach(fun({Key, Value}) ->
ok = erlfdb:set(Tx, Key, Value)
- end, doc_to_fdb(Db, Doc)).
+ end, Rows),
+ {ok, RevSize}.
clear_doc_body(_Db, _DocId, not_found) ->
@@ -1123,7 +1171,8 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
rev_id := {RevPos, Rev},
rev_path := RevPath,
branch_count := BranchCount,
- att_hash := AttHash
+ att_hash := AttHash,
+ rev_size := RevSize
} = RevId,
VS = new_versionstamp(Tx),
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
@@ -1132,7 +1181,8 @@ revinfo_to_fdb(Tx, DbPrefix, DocId, #{winner := true} = RevId) ->
VS,
BranchCount,
list_to_tuple(RevPath),
- AttHash
+ AttHash,
+ RevSize
},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack_vs(Val),
@@ -1143,18 +1193,19 @@ revinfo_to_fdb(_Tx, DbPrefix, DocId, #{} = RevId) ->
deleted := Deleted,
rev_id := {RevPos, Rev},
rev_path := RevPath,
- att_hash := AttHash
+ att_hash := AttHash,
+ rev_size := RevSize
} = RevId,
Key = {?DB_REVS, DocId, not Deleted, RevPos, Rev},
- Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), AttHash},
+ Val = {?CURR_REV_FORMAT, list_to_tuple(RevPath), AttHash, RevSize},
KBin = erlfdb_tuple:pack(Key, DbPrefix),
VBin = erlfdb_tuple:pack(Val),
{KBin, VBin, undefined}.
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, Sequence, BranchCount, RevPath, AttHash} = Val,
+ {_RevFormat, Sequence, BranchCount, RevPath, AttHash, RevSize} = Val,
#{
winner => true,
deleted => not NotDeleted,
@@ -1162,12 +1213,13 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _, _} = Val) ->
rev_path => tuple_to_list(RevPath),
sequence => Sequence,
branch_count => BranchCount,
- att_hash => AttHash
+ att_hash => AttHash,
+ rev_size => RevSize
};
-fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val) ->
+fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _, _} = Val) ->
{?DB_REVS, _DocId, NotDeleted, RevPos, Rev} = Key,
- {_RevFormat, RevPath, AttHash} = Val,
+ {_RevFormat, RevPath, AttHash, RevSize} = Val,
#{
winner => false,
deleted => not NotDeleted,
@@ -1175,7 +1227,8 @@ fdb_to_revinfo(Key, {?CURR_REV_FORMAT, _, _} = Val) ->
rev_path => tuple_to_list(RevPath),
sequence => undefined,
branch_count => undefined,
- att_hash => AttHash
+ att_hash => AttHash,
+ rev_size => RevSize
};
fdb_to_revinfo(Key, {0, Seq, BCount, RPath}) ->
@@ -1184,6 +1237,14 @@ fdb_to_revinfo(Key, {0, Seq, BCount, RPath}) ->
fdb_to_revinfo(Key, {0, RPath}) ->
Val = {?CURR_REV_FORMAT, RPath, <<>>},
+ fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {1, Seq, BCount, RPath, AttHash}) ->
+ Val = {?CURR_REV_FORMAT, Seq, BCount, RPath, AttHash, null},
+ fdb_to_revinfo(Key, Val);
+
+fdb_to_revinfo(Key, {1, RPath, AttHash}) ->
+ Val = {?CURR_REV_FORMAT, RPath, AttHash, null},
fdb_to_revinfo(Key, Val).
@@ -1203,12 +1264,26 @@ doc_to_fdb(Db, #doc{} = Doc) ->
DiskAtts = lists:map(fun couch_att:to_disk_term/1, Atts),
Value = term_to_binary({Body, DiskAtts, Deleted}, [{minor_version, 1}]),
+ Chunks = chunkify_binary(Value),
{Rows, _} = lists:mapfoldl(fun(Chunk, ChunkId) ->
Key = erlfdb_tuple:pack({?DB_DOCS, Id, Start, Rev, ChunkId}, DbPrefix),
{{Key, Chunk}, ChunkId + 1}
- end, 0, chunkify_binary(Value)),
- Rows.
+ end, 0, Chunks),
+
+ % Calculate the size of this revision
+ TotalSize = lists:sum([
+ size(Id),
+ size(erlfdb_tuple:pack({Start})),
+ size(Rev),
+ 1, % FDB tuple encoding of booleans for deleted flag is 1 byte
+ couch_ejson_size:encoded_size(Body),
+ lists:foldl(fun(Att, Acc) ->
+ couch_att:external_size(Att) + Acc
+ end, 0, Atts)
+ ]),
+
+ {Rows, TotalSize}.
fdb_to_doc(_Db, _DocId, _Pos, _Path, []) ->
@@ -1258,9 +1333,29 @@ local_doc_to_fdb(Db, #doc{} = Doc) ->
{{K, Chunk}, ChunkId + 1}
end, 0, chunkify_binary(BVal)),
- {Key, StoreRev, Rows}.
+ % Calculate size
+ TotalSize = case Doc#doc.deleted of
+ true ->
+ 0;
+ false ->
+ lists:sum([
+ size(Id),
+ size(StoreRev),
+ couch_ejson_size:encoded_size(Body)
+ ])
+ end,
+
+ RawValue = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, StoreRev, TotalSize}),
+
+ % Prefix our tuple encoding to make upgrades easier
+ Value = <<255, RawValue/binary>>,
+
+ {Key, Value, TotalSize, Rows}.
+fdb_to_local_doc(_Db, _DocId, not_found, []) ->
+ {not_found, missing};
+
fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
% This is an upgrade clause for the old encoding. We allow reading the old
% value and will perform an upgrade of the storage format on an update.
@@ -1272,18 +1367,25 @@ fdb_to_local_doc(_Db, DocId, <<131, _/binary>> = Val, []) ->
body = Body
};
-fdb_to_local_doc(_Db, _DocId, not_found, []) ->
- {not_found, missing};
+fdb_to_local_doc(_Db, DocId, <<255, RevBin/binary>>, Rows) when is_list(Rows) ->
+ Rev = case erlfdb_tuple:unpack(RevBin) of
+ {?CURR_LDOC_FORMAT, Rev0, _Size} -> Rev0
+ end,
-fdb_to_local_doc(_Db, DocId, Rev, Rows) when is_list(Rows), is_binary(Rev) ->
BodyBin = iolist_to_binary(Rows),
Body = binary_to_term(BodyBin, [safe]),
+
#doc{
id = DocId,
revs = {0, [Rev]},
deleted = false,
body = Body
- }.
+ };
+
+fdb_to_local_doc(Db, DocId, RawRev, Rows) ->
+ BaseRev = erlfdb_tuple:pack({?CURR_LDOC_FORMAT, RawRev, 0}),
+ Rev = <<255, BaseRev/binary>>,
+ fdb_to_local_doc(Db, DocId, Rev, Rows).
chunkify_binary(Data) ->
diff --git a/src/fabric/test/fabric2_doc_crud_tests.erl b/src/fabric/test/fabric2_doc_crud_tests.erl
index 184eb4a..46cd4fc 100644
--- a/src/fabric/test/fabric2_doc_crud_tests.erl
+++ b/src/fabric/test/fabric2_doc_crud_tests.erl
@@ -884,11 +884,12 @@ local_doc_with_previous_encoding({Db, _}) ->
?assertEqual(NewBody, Doc3#doc.body),
% Old doc now has only the rev number in it
- OldDocBin = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ <<255, OldDocBin/binary>> = fabric2_fdb:transactional(Db, fun(TxDb) ->
#{tx := Tx} = TxDb,
erlfdb:wait(erlfdb:get(Tx, Key))
end),
- ?assertEqual(<<"2">> , OldDocBin).
+ Unpacked = erlfdb_tuple:unpack(OldDocBin),
+ ?assertMatch({?CURR_LDOC_FORMAT, <<"2">>, _}, Unpacked).
before_doc_update_skips_local_docs({Db0, _}) ->
[couchdb] 10/16: Implement `fabric2_db:list_dbs_info/1,2,3`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b2c69381bea35d714d7e2e3cfaa0dce822491154
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:45:36 2019 -0600
Implement `fabric2_db:list_dbs_info/1,2,3`
This API allows for listing all database info blobs in a single request.
It accepts the same parameters as `_all_dbs` for controlling pagination
of results and so on.
---
src/fabric/src/fabric2_db.erl | 100 +++++++++++++++++++++++++-----
src/fabric/src/fabric2_fdb.erl | 11 ++++
src/fabric/test/fabric2_db_crud_tests.erl | 31 ++++++++-
3 files changed, 126 insertions(+), 16 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 6d015df..17c899d 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -22,6 +22,10 @@
list_dbs/1,
list_dbs/3,
+ list_dbs_info/0,
+ list_dbs_info/1,
+ list_dbs_info/3,
+
check_is_admin/1,
check_is_member/1,
@@ -238,6 +242,46 @@ list_dbs(UserFun, UserAcc0, Options) ->
end).
+list_dbs_info() ->
+ list_dbs_info([]).
+
+
+list_dbs_info(Options) ->
+ Callback = fun(Value, Acc) ->
+ NewAcc = case Value of
+ {meta, _} -> Acc;
+ {row, DbInfo} -> [DbInfo | Acc];
+ complete -> Acc
+ end,
+ {ok, NewAcc}
+ end,
+ {ok, DbInfos} = list_dbs_info(Callback, [], Options),
+ {ok, lists:reverse(DbInfos)}.
+
+
+list_dbs_info(UserFun, UserAcc0, Options) ->
+ FoldFun = fun(DbName, InfoFuture, {FutureQ, Count, Acc}) ->
+ NewFutureQ = queue:in({DbName, InfoFuture}, FutureQ),
+ drain_info_futures(NewFutureQ, Count + 1, UserFun, Acc)
+ end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ try
+ UserAcc1 = maybe_stop(UserFun({meta, []}, UserAcc0)),
+ InitAcc = {queue:new(), 0, UserAcc1},
+ {FinalFutureQ, _, UserAcc2} = fabric2_fdb:list_dbs_info(
+ Tx,
+ FoldFun,
+ InitAcc,
+ Options
+ ),
+ UserAcc3 = drain_all_info_futures(FinalFutureQ, UserFun, UserAcc2),
+ {ok, maybe_stop(UserFun(complete, UserAcc3))}
+ catch throw:{stop, FinalUserAcc} ->
+ {ok, FinalUserAcc}
+ end
+ end).
+
+
is_admin(Db, {SecProps}) when is_list(SecProps) ->
case fabric2_db_plugin:check_is_admin(Db) of
true ->
@@ -313,21 +357,7 @@ get_db_info(#{} = Db) ->
DbProps = fabric2_fdb:transactional(Db, fun(TxDb) ->
fabric2_fdb:get_info(TxDb)
end),
-
- BaseProps = [
- {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
- {compact_running, false},
- {data_size, 0},
- {db_name, name(Db)},
- {disk_format_version, 0},
- {disk_size, 0},
- {instance_start_time, <<"0">>},
- {purge_seq, 0}
- ],
-
- {ok, lists:foldl(fun({Key, Val}, Acc) ->
- lists:keystore(Key, 1, Acc, {Key, Val})
- end, BaseProps, DbProps)}.
+ {ok, make_db_info(name(Db), DbProps)}.
get_del_doc_count(#{} = Db) ->
@@ -944,6 +974,46 @@ maybe_add_sys_db_callbacks(Db) ->
}.
+make_db_info(DbName, Props) ->
+ BaseProps = [
+ {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
+ {compact_running, false},
+ {data_size, 0},
+ {db_name, DbName},
+ {disk_format_version, 0},
+ {disk_size, 0},
+ {instance_start_time, <<"0">>},
+ {purge_seq, 0}
+ ],
+
+ lists:foldl(fun({Key, Val}, Acc) ->
+ lists:keystore(Key, 1, Acc, {Key, Val})
+ end, BaseProps, Props).
+
+
+drain_info_futures(FutureQ, Count, _UserFun, Acc) when Count < 100 ->
+ {FutureQ, Count, Acc};
+
+drain_info_futures(FutureQ, Count, UserFun, Acc) when Count >= 100 ->
+ {{value, {DbName, Future}}, RestQ} = queue:out(FutureQ),
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ {RestQ, Count - 1, NewAcc}.
+
+
+drain_all_info_futures(FutureQ, UserFun, Acc) ->
+ case queue:out(FutureQ) of
+ {{value, {DbName, Future}}, RestQ} ->
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ drain_all_info_futures(RestQ, UserFun, NewAcc);
+ {empty, _} ->
+ Acc
+ end.
+
+
new_revid(Db, Doc) ->
#doc{
id = DocId,
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 0e7cba8..99611b0 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -27,6 +27,7 @@
get_dir/1,
list_dbs/4,
+ list_dbs_info/4,
get_info/1,
get_info_future/2,
@@ -330,6 +331,16 @@ list_dbs(Tx, Callback, AccIn, Options) ->
end, AccIn, Options).
+list_dbs_info(Tx, Callback, AccIn, Options) ->
+ LayerPrefix = get_dir(Tx),
+ Prefix = erlfdb_tuple:pack({?ALL_DBS}, LayerPrefix),
+ fold_range({tx, Tx}, Prefix, fun({DbNameKey, DbPrefix}, Acc) ->
+ {DbName} = erlfdb_tuple:unpack(DbNameKey, Prefix),
+ InfoFuture = get_info_future(Tx, DbPrefix),
+ Callback(DbName, InfoFuture, Acc)
+ end, AccIn, Options).
+
+
get_info(#{} = Db) ->
#{
tx := Tx,
diff --git a/src/fabric/test/fabric2_db_crud_tests.erl b/src/fabric/test/fabric2_db_crud_tests.erl
index cc44f7d..8052551 100644
--- a/src/fabric/test/fabric2_db_crud_tests.erl
+++ b/src/fabric/test/fabric2_db_crud_tests.erl
@@ -29,7 +29,8 @@ crud_test_() ->
?TDEF(create_db),
?TDEF(open_db),
?TDEF(delete_db),
- ?TDEF(list_dbs)
+ ?TDEF(list_dbs),
+ ?TDEF(list_dbs_info)
])
}
}.
@@ -84,3 +85,31 @@ list_dbs(_) ->
?assertEqual(ok, fabric2_db:delete(DbName, [])),
AllDbs3 = fabric2_db:list_dbs(),
?assert(not lists:member(DbName, AllDbs3)).
+
+
+list_dbs_info(_) ->
+ DbName = ?tempdb(),
+ {ok, AllDbInfos1} = fabric2_db:list_dbs_info(),
+
+ ?assert(is_list(AllDbInfos1)),
+ ?assert(not is_db_info_member(DbName, AllDbInfos1)),
+
+ ?assertMatch({ok, _}, fabric2_db:create(DbName, [])),
+ {ok, AllDbInfos2} = fabric2_db:list_dbs_info(),
+ ?assert(is_db_info_member(DbName, AllDbInfos2)),
+
+ ?assertEqual(ok, fabric2_db:delete(DbName, [])),
+ {ok, AllDbInfos3} = fabric2_db:list_dbs_info(),
+ ?assert(not is_db_info_member(DbName, AllDbInfos3)).
+
+
+is_db_info_member(_, []) ->
+ false;
+
+is_db_info_member(DbName, [DbInfo | RestInfos]) ->
+ case lists:keyfind(db_name, 1, DbInfo) of
+ {db_name, DbName} ->
+ true;
+ _E ->
+ is_db_info_member(DbName, RestInfos)
+ end.
[couchdb] 05/16: Delete unused ets table creation
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a7068faf65bfd5087564dbc18a2665bbbc2ca308
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Jan 28 19:06:07 2020 -0800
Delete unused ets table creation
This ets table was a holdover from when couch_expiring_cache was a non-
library OTP application. It is unused, and would prevent multiple users
of the library in the same project.
---
src/couch_expiring_cache/src/couch_expiring_cache_server.erl | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
index 6f9dc1f..65e742b 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -43,7 +43,6 @@ start_link(Name, Opts) when is_atom(Name) ->
init(Opts) ->
- ?MODULE = ets:new(?MODULE, [named_table, public, {read_concurrency, true}]),
DefaultCacheName = atom_to_binary(maps:get(name, Opts), utf8),
Period = maps:get(period, Opts, ?DEFAULT_PERIOD_MSEC),
MaxJitter = maps:get(max_jitter, Opts, ?DEFAULT_MAX_JITTER_MSEC),
[couchdb] 07/16: Support jaeger http reporter
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3565e524cb50c8596dc9739907e3a75dc3151d80
Author: ILYA Khlopotov <ii...@apache.org>
AuthorDate: Fri Jan 17 09:37:54 2020 -0800
Support jaeger http reporter
---
rebar.config.script | 4 ++--
rel/overlay/etc/default.ini | 12 +++++++++---
src/ctrace/README.md | 18 ++++++++++++++++--
src/ctrace/src/ctrace_config.erl | 38 +++++++++++++++++++++++++++++---------
4 files changed, 56 insertions(+), 16 deletions(-)
diff --git a/rebar.config.script b/rebar.config.script
index 2fde9c5..f7ebcb5 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -121,13 +121,13 @@ DepDescs = [
{folsom, "folsom", {tag, "CouchDB-0.8.3"}},
{hyper, "hyper", {tag, "CouchDB-2.2.0-4"}},
{ibrowse, "ibrowse", {tag, "CouchDB-4.0.1-1"}},
-{jaeger_passage, "jaeger-passage", {tag, "CouchDB-0.1.13-1"}},
+{jaeger_passage, "jaeger-passage", {tag, "CouchDB-0.1.14-1"}},
{jiffy, "jiffy", {tag, "CouchDB-0.14.11-2"}},
{local, "local", {tag, "0.2.1"}},
{mochiweb, "mochiweb", {tag, "v2.19.0"}},
{meck, "meck", {tag, "0.8.8"}},
{passage, "passage", {tag, "0.2.6"}},
-{thrift_protocol, "thrift-protocol", {tag, "0.1.3"}},
+{thrift_protocol, "thrift-protocol", {tag, "0.1.5"}},
%% TMP - Until this is moved to a proper Apache repo
{erlfdb, "erlfdb", {branch, "master"}}
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 63cb443..592445c 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -568,13 +568,19 @@ min_priority = 2.0
[tracing]
;
; Configuration settings for the `ctrace` OpenTracing
-; API.
-;
+; API. There are two reporter which we support.
+; - jaeger.thrift over udp
+; - jaeger.thrift over http
+; ## Common settings
; enabled = false ; true | false
+; app_name = couchdb ; value to use for the `location.application` tag
+; protocol = udp ; udp | http - which reporter to use
+; ## jaeger.thrift over udp reporter
; thrift_format = compact ; compact | binary
; agent_host = 127.0.0.1
; agent_port = 6831
-; app_name = couchdb ; value to use for the `location.application` tag
+; ## jaeger.thrift over udp reporter
+; endpoint = http://127.0.0.1:14268
[tracing.filters]
;
diff --git a/src/ctrace/README.md b/src/ctrace/README.md
index 3172f26..4b0238b 100644
--- a/src/ctrace/README.md
+++ b/src/ctrace/README.md
@@ -120,9 +120,23 @@ Configuration
Traces are configured using standard CouchDB ini file based configuration.
There is a global toggle `[tracing] enabled = true | false` that switches
tracing on or off completely. The `[tracing]` section also includes
-configuration for where to send trace data.
+configuration for where to send trace data. There are two reporters which we
+support.
-An example `[tracing]` section
+The thrift over udp reporter (this is the default) has following configuration
+options:
+
+- protocol = udp
+- thrift_format = compact | binary
+- agent_host = 127.0.0.1
+- agent_port = 6831
+
+The thrift over http has following options
+
+- protocol = http
+- endpoint = http://127.0.0.1:14268
+
+An example of `[tracing]` section
```ini
[tracing]
diff --git a/src/ctrace/src/ctrace_config.erl b/src/ctrace/src/ctrace_config.erl
index bc2a3df..c63c77f 100644
--- a/src/ctrace/src/ctrace_config.erl
+++ b/src/ctrace/src/ctrace_config.erl
@@ -98,17 +98,37 @@ maybe_start_main_tracer(TracerId) ->
start_main_tracer(TracerId) ->
- Sampler = passage_sampler_all:new(),
- Options = [
- {thrift_format,
- list_to_atom(config:get("tracing", "thrift_format", "compact"))},
- {agent_host, config:get("tracing", "agent_host", "127.0.0.1")},
- {agent_port, config:get_integer("tracing", "agent_port", 6831)},
- {default_service_name,
- list_to_atom(config:get("tracing", "app_name", "couchdb"))}
- ],
+ MaxQueueLen = config:get_integer("tracing", "max_queue_len", 1024),
+ Sampler = jaeger_passage_sampler_queue_limit:new(
+ passage_sampler_all:new(), TracerId, MaxQueueLen),
+ ServiceName = list_to_atom(config:get("tracing", "app_name", "couchdb")),
+
+ ProtocolOptions = case config:get("tracing", "protocol", "udp") of
+ "udp" ->
+ [
+ {thrift_format, list_to_atom(
+ config:get("tracing", "thrift_format", "compact"))},
+ {agent_host,
+ config:get("tracing", "agent_host", "127.0.0.1")},
+ {agent_port,
+ config:get_integer("tracing", "agent_port", 6831)},
+ {protocol, udp},
+ {default_service_name, ServiceName}
+ ];
+ "http" ++ _ ->
+ [
+ {endpoint,
+ config:get("tracing", "endpoint", "http://127.0.0.1:14268")},
+ {protocol, http},
+ {http_client, fun http_client/5},
+ {default_service_name, ServiceName}
+ ]
+ end,
+ Options = [{default_service_name, ServiceName}|ProtocolOptions],
ok = jaeger_passage:start_tracer(TracerId, Sampler, Options).
+http_client(Endpoint, Method, Headers, Body, _ReporterOptions) ->
+ ibrowse:send_req(Endpoint, Headers, Method, Body, []).
compile_filter(OperationId, FilterDef) ->
try