You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/12/03 19:58:09 UTC
[couchdb] branch prototype/fdb-layer-get-dbs-info updated (a2f9eea
-> 771acd5)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard a2f9eea Support `GET /_dbs_info` endpoint
discard 227d2ca Implement `fabric2_db:list_dbs_info/1,2,3`
discard ce5a6c7 Implement async API for `fabric2_fdb:get_info/1`
discard 647bf90 Expose rolled up view size in dbinfo blobs
discard 614acee Track a database level view size rollup
new f4a60cf Track a database level view size rollup
new ab13a0e Implement async API for `fabric2_fdb:get_info/1`
new b27e2de Implement `fabric2_db:list_dbs_info/1,2,3`
new 771acd5 Support `GET /_dbs_info` endpoint
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a2f9eea)
\
N -- N -- N refs/heads/prototype/fdb-layer-get-dbs-info (771acd5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch_views/src/couch_views_fdb.erl | 2 +-
src/fabric/src/fabric2_fdb.erl | 23 +++++++++++------------
2 files changed, 12 insertions(+), 13 deletions(-)
[couchdb] 03/04: Implement `fabric2_db:list_dbs_info/1,2,3`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b27e2de177f31b6444190e71e2f8525153e6e003
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:45:36 2019 -0600
Implement `fabric2_db:list_dbs_info/1,2,3`
This API allows for listing all database info blobs in a single request.
It accepts the same parameters as `_all_dbs` for controlling pagination
of results and so on.
---
src/fabric/src/fabric2_db.erl | 100 +++++++++++++++++++++++++-----
src/fabric/src/fabric2_fdb.erl | 11 ++++
src/fabric/test/fabric2_db_crud_tests.erl | 31 ++++++++-
3 files changed, 126 insertions(+), 16 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index 88840e7..688f794 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -22,6 +22,10 @@
list_dbs/1,
list_dbs/3,
+ list_dbs_info/0,
+ list_dbs_info/1,
+ list_dbs_info/3,
+
check_is_admin/1,
check_is_member/1,
@@ -238,6 +242,46 @@ list_dbs(UserFun, UserAcc0, Options) ->
end).
+list_dbs_info() ->
+ list_dbs_info([]).
+
+
+list_dbs_info(Options) ->
+ Callback = fun(Value, Acc) ->
+ NewAcc = case Value of
+ {meta, _} -> Acc;
+ {row, DbInfo} -> [DbInfo | Acc];
+ complete -> Acc
+ end,
+ {ok, NewAcc}
+ end,
+ {ok, DbInfos} = list_dbs_info(Callback, [], Options),
+ {ok, lists:reverse(DbInfos)}.
+
+
+list_dbs_info(UserFun, UserAcc0, Options) ->
+ FoldFun = fun(DbName, InfoFuture, {FutureQ, Count, Acc}) ->
+ NewFutureQ = queue:in({DbName, InfoFuture}, FutureQ),
+ drain_info_futures(NewFutureQ, Count + 1, UserFun, Acc)
+ end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ try
+ UserAcc1 = maybe_stop(UserFun({meta, []}, UserAcc0)),
+ InitAcc = {queue:new(), 0, UserAcc1},
+ {FinalFutureQ, _, UserAcc2} = fabric2_fdb:list_dbs_info(
+ Tx,
+ FoldFun,
+ InitAcc,
+ Options
+ ),
+ UserAcc3 = drain_all_info_futures(FinalFutureQ, UserFun, UserAcc2),
+ {ok, maybe_stop(UserFun(complete, UserAcc3))}
+ catch throw:{stop, FinalUserAcc} ->
+ {ok, FinalUserAcc}
+ end
+ end).
+
+
is_admin(Db, {SecProps}) when is_list(SecProps) ->
case fabric2_db_plugin:check_is_admin(Db) of
true ->
@@ -313,21 +357,7 @@ get_db_info(#{} = Db) ->
DbProps = fabric2_fdb:transactional(Db, fun(TxDb) ->
fabric2_fdb:get_info(TxDb)
end),
-
- BaseProps = [
- {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
- {compact_running, false},
- {data_size, 0},
- {db_name, name(Db)},
- {disk_format_version, 0},
- {disk_size, 0},
- {instance_start_time, <<"0">>},
- {purge_seq, 0}
- ],
-
- {ok, lists:foldl(fun({Key, Val}, Acc) ->
- lists:keystore(Key, 1, Acc, {Key, Val})
- end, BaseProps, DbProps)}.
+ {ok, make_db_info(name(Db), DbProps)}.
get_del_doc_count(#{} = Db) ->
@@ -944,6 +974,46 @@ maybe_add_sys_db_callbacks(Db) ->
}.
+make_db_info(DbName, Props) ->
+ BaseProps = [
+ {cluster, {[{n, 0}, {q, 0}, {r, 0}, {w, 0}]}},
+ {compact_running, false},
+ {data_size, 0},
+ {db_name, DbName},
+ {disk_format_version, 0},
+ {disk_size, 0},
+ {instance_start_time, <<"0">>},
+ {purge_seq, 0}
+ ],
+
+ lists:foldl(fun({Key, Val}, Acc) ->
+ lists:keystore(Key, 1, Acc, {Key, Val})
+ end, BaseProps, Props).
+
+
+drain_info_futures(FutureQ, Count, _UserFun, Acc) when Count < 100 ->
+ {FutureQ, Count, Acc};
+
+drain_info_futures(FutureQ, Count, UserFun, Acc) when Count >= 100 ->
+ {{value, {DbName, Future}}, RestQ} = queue:out(FutureQ),
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ {RestQ, Count - 1, NewAcc}.
+
+
+drain_all_info_futures(FutureQ, UserFun, Acc) ->
+ case queue:out(FutureQ) of
+ {{value, {DbName, Future}}, RestQ} ->
+ InfoProps = fabric2_fdb:get_info_wait(Future),
+ DbInfo = make_db_info(DbName, InfoProps),
+ NewAcc = maybe_stop(UserFun({row, DbInfo}, Acc)),
+ drain_all_info_futures(RestQ, UserFun, NewAcc);
+ {empty, _} ->
+ Acc
+ end.
+
+
new_revid(Db, Doc) ->
#doc{
id = DocId,
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 17d3c14..c13ef7c 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -27,6 +27,7 @@
get_dir/1,
list_dbs/4,
+ list_dbs_info/4,
get_info/1,
get_info_future/2,
@@ -327,6 +328,16 @@ list_dbs(Tx, Callback, AccIn, Options) ->
end, AccIn, Options).
+list_dbs_info(Tx, Callback, AccIn, Options) ->
+ LayerPrefix = get_dir(Tx),
+ Prefix = erlfdb_tuple:pack({?ALL_DBS}, LayerPrefix),
+ fold_range({tx, Tx}, Prefix, fun({DbNameKey, DbPrefix}, Acc) ->
+ {DbName} = erlfdb_tuple:unpack(DbNameKey, Prefix),
+ InfoFuture = get_info_future(Tx, DbPrefix),
+ Callback(DbName, InfoFuture, Acc)
+ end, AccIn, Options).
+
+
get_info(#{} = Db) ->
#{
tx := Tx,
diff --git a/src/fabric/test/fabric2_db_crud_tests.erl b/src/fabric/test/fabric2_db_crud_tests.erl
index 24deeb2..34d16f3 100644
--- a/src/fabric/test/fabric2_db_crud_tests.erl
+++ b/src/fabric/test/fabric2_db_crud_tests.erl
@@ -31,7 +31,8 @@ crud_test_() ->
?TDEF(create_db),
?TDEF(open_db),
?TDEF(delete_db),
- ?TDEF(list_dbs)
+ ?TDEF(list_dbs),
+ ?TDEF(list_dbs_info)
]
}
}.
@@ -86,3 +87,31 @@ list_dbs() ->
?assertEqual(ok, fabric2_db:delete(DbName, [])),
AllDbs3 = fabric2_db:list_dbs(),
?assert(not lists:member(DbName, AllDbs3)).
+
+
+list_dbs_info() ->
+ DbName = ?tempdb(),
+ {ok, AllDbInfos1} = fabric2_db:list_dbs_info(),
+
+ ?assert(is_list(AllDbInfos1)),
+ ?assert(not is_db_info_member(DbName, AllDbInfos1)),
+
+ ?assertMatch({ok, _}, fabric2_db:create(DbName, [])),
+ {ok, AllDbInfos2} = fabric2_db:list_dbs_info(),
+ ?assert(is_db_info_member(DbName, AllDbInfos2)),
+
+ ?assertEqual(ok, fabric2_db:delete(DbName, [])),
+ {ok, AllDbInfos3} = fabric2_db:list_dbs_info(),
+ ?assert(not is_db_info_member(DbName, AllDbInfos3)).
+
+
+is_db_info_member(_, []) ->
+ false;
+
+is_db_info_member(DbName, [DbInfo | RestInfos]) ->
+ case lists:keyfind(db_name, 1, DbInfo) of
+ {db_name, DbName} ->
+ true;
+ _E ->
+ is_db_info_member(DbName, RestInfos)
+ end.
[couchdb] 04/04: Support `GET /_dbs_info` endpoint
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 771acd5ae8837dbc889b93135a1d08eeb7456732
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 13:44:35 2019 -0600
Support `GET /_dbs_info` endpoint
Previously only `POST` with a list of keys was supported. The new `GET`
support just dumps all database info blobs in a single ordered response.
---
src/chttpd/src/chttpd_misc.erl | 52 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 51 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl
index 186ec9f..21a4fef 100644
--- a/src/chttpd/src/chttpd_misc.erl
+++ b/src/chttpd/src/chttpd_misc.erl
@@ -157,6 +157,40 @@ all_dbs_callback({error, Reason}, #vacc{resp=Resp0}=Acc) ->
{ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
{ok, Acc#vacc{resp=Resp1}}.
+handle_dbs_info_req(#httpd{method = 'GET'} = Req) ->
+ ok = chttpd:verify_is_server_admin(Req),
+
+ #mrargs{
+ start_key = StartKey,
+ end_key = EndKey,
+ direction = Dir,
+ limit = Limit,
+ skip = Skip
+ } = couch_mrview_http:parse_params(Req, undefined),
+
+ Options = [
+ {start_key, StartKey},
+ {end_key, EndKey},
+ {dir, Dir},
+ {limit, Limit},
+ {skip, Skip}
+ ],
+
+ % Eventually the Etag for this request will be derived
+ % from the \xFFmetadataVersion key in fdb
+ Etag = <<"foo">>,
+
+ {ok, Resp} = chttpd:etag_respond(Req, Etag, fun() ->
+ Headers = [{"ETag", Etag}],
+ {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
+ Callback = fun dbs_info_callback/2,
+ Acc = #vacc{req = Req, resp = Resp},
+ fabric2_db:list_dbs_info(Callback, Acc, Options)
+ end),
+ case is_record(Resp, vacc) of
+ true -> {ok, Resp#vacc.resp};
+ _ -> {ok, Resp}
+ end;
handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
chttpd:validate_ctype(Req, "application/json"),
Props = chttpd:json_body_obj(Req),
@@ -188,7 +222,23 @@ handle_dbs_info_req(#httpd{method='POST', user_ctx=UserCtx}=Req) ->
send_chunk(Resp, "]"),
chttpd:end_json_response(Resp);
handle_dbs_info_req(Req) ->
- send_method_not_allowed(Req, "POST").
+ send_method_not_allowed(Req, "GET,HEAD,POST").
+
+dbs_info_callback({meta, _Meta}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "["),
+ {ok, Acc#vacc{resp = Resp1}};
+dbs_info_callback({row, Props}, #vacc{resp = Resp0} = Acc) ->
+ Prepend = couch_mrview_http:prepend_val(Acc),
+ Chunk = [Prepend, ?JSON_ENCODE({Props})],
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+ {ok, Acc#vacc{prepend = ",", resp = Resp1}};
+dbs_info_callback(complete, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "]"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, Acc#vacc{resp = Resp2}};
+dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) ->
+ {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason),
+ {ok, Acc#vacc{resp = Resp1}}.
handle_task_status_req(#httpd{method='GET'}=Req) ->
ok = chttpd:verify_is_server_admin(Req),
[couchdb] 02/04: Implement async API for `fabric2_fdb:get_info/1`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ab13a0e3ecda4333d23a09d3c562f3a8d9c9029c
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 12:44:34 2019 -0600
Implement async API for `fabric2_fdb:get_info/1`
---
src/fabric/src/fabric2_fdb.erl | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 1f68b68..17d3c14 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -29,6 +29,8 @@
list_dbs/4,
get_info/1,
+ get_info_future/2,
+ get_info_wait/1,
set_config/3,
get_stat/2,
@@ -330,7 +332,10 @@ get_info(#{} = Db) ->
tx := Tx,
db_prefix := DbPrefix
} = ensure_current(Db),
+ get_info_wait(get_info_future(Tx, DbPrefix)).
+
+get_info_future(Tx, DbPrefix) ->
{CStart, CEnd} = erlfdb_tuple:range({?DB_CHANGES}, DbPrefix),
ChangesFuture = erlfdb:get_range(Tx, CStart, CEnd, [
{streaming_mode, exact},
@@ -341,6 +346,10 @@ get_info(#{} = Db) ->
StatsPrefix = erlfdb_tuple:pack({?DB_STATS}, DbPrefix),
MetaFuture = erlfdb:get_range_startswith(Tx, StatsPrefix),
+ {DbPrefix, ChangesFuture, MetaFuture}.
+
+
+get_info_wait({DbPrefix, ChangesFuture, MetaFuture}) ->
RawSeq = case erlfdb:wait(ChangesFuture) of
[] ->
vs_to_seq(fabric2_util:seq_zero_vs());
[couchdb] 01/04: Track a database level view size rollup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-get-dbs-info
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f4a60cf4830d1fda150a4de5003e96d91da19040
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Dec 3 10:24:36 2019 -0600
Track a database level view size rollup
This way we can expose the total view size for a database in the dbinfo
JSON blob.
---
src/couch_views/src/couch_views_fdb.erl | 17 ++++++++++++++--
src/fabric/src/fabric2_fdb.erl | 36 ++++++++++++++++-----------------
2 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 60ce300..1972e22 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -366,8 +366,16 @@ update_kv_size(TxDb, Sig, ViewId, Increment) ->
tx := Tx,
db_prefix := DbPrefix
} = TxDb,
- Key = kv_size_key(DbPrefix, Sig, ViewId),
- erlfdb:add(Tx, Key, Increment).
+
+ % Track a view specific size for calls to
+ % GET /dbname/_design/doc/_info`
+ IdxKey = kv_size_key(DbPrefix, Sig, ViewId),
+ erlfdb:add(Tx, IdxKey, Increment),
+
+ % Track a database level rollup for calls to
+ % GET /dbname
+ DbKey = db_kv_size_key(DbPrefix),
+ erlfdb:add(Tx, DbKey, Increment).
seq_key(DbPrefix, Sig) ->
@@ -385,6 +393,11 @@ kv_size_key(DbPrefix, Sig, ViewId) ->
erlfdb_tuple:pack(Key, DbPrefix).
+db_kv_size_key(DbPrefix) ->
+ Key = {?DB_STATS, <<"sizes">>, <<"views">>},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index fb2891b..1f68b68 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -171,11 +171,16 @@ create(#{} = Db0, Options) ->
{?DB_STATS, <<"doc_del_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_design_count">>, ?uint2bin(0)},
{?DB_STATS, <<"doc_local_count">>, ?uint2bin(0)},
- {?DB_STATS, <<"size">>, ?uint2bin(2)}
+ {?DB_STATS, <<"sizes">>, <<"external">>, ?uint2bin(2)},
+ {?DB_STATS, <<"sizes">>, <<"views">>, ?uint2bin(0)}
],
- lists:foreach(fun({P, K, V}) ->
- Key = erlfdb_tuple:pack({P, K}, DbPrefix),
- erlfdb:set(Tx, Key, V)
+ lists:foreach(fun
+ ({P, K, V}) ->
+ Key = erlfdb_tuple:pack({P, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V);
+ ({P, S, K, V}) ->
+ Key = erlfdb_tuple:pack({P, S, K}, DbPrefix),
+ erlfdb:set(Tx, Key, V)
end, Defaults),
UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}),
@@ -345,26 +350,21 @@ get_info(#{} = Db) ->
end,
CProp = {update_seq, RawSeq},
- MProps = lists:flatmap(fun({K, V}) ->
+ MProps = lists:foldl(fun({K, V}, Acc) ->
case erlfdb_tuple:unpack(K, DbPrefix) of
{?DB_STATS, <<"doc_count">>} ->
- [{doc_count, ?bin2uint(V)}];
+ [{doc_count, ?bin2uint(V)} | Acc];
{?DB_STATS, <<"doc_del_count">>} ->
- [{doc_del_count, ?bin2uint(V)}];
- {?DB_STATS, <<"size">>} ->
+ [{doc_del_count, ?bin2uint(V)} | Acc];
+ {?DB_STATS, <<"sizes">>, Name} ->
Val = ?bin2uint(V),
- [
- {other, {[{data_size, Val}]}},
- {sizes, {[
- {active, 0},
- {external, Val},
- {file, 0}
- ]}}
- ];
+ {_, {Sizes}} = lists:keyfind(sizes, 1, Acc),
+ NewSizes = lists:keystore(Name, 1, Sizes, {Name, Val}),
+ lists:keystore(sizes, 1, Acc, {sizes, {NewSizes}});
{?DB_STATS, _} ->
- []
+ Acc
end
- end, erlfdb:wait(MetaFuture)),
+ end, [{sizes, {[]}}], erlfdb:wait(MetaFuture)),
[CProp | MProps].