You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2018/09/20 12:38:23 UTC
[couchdb] branch add-partition-info-endpoint created (now fda7ee8)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch add-partition-info-endpoint
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at fda7ee8 Add /_partition/:partition/_info endpoint
This branch includes the following new commits:
new fda7ee8 Add /_partition/:partition/_info endpoint
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Add /_partition/:partition/_info endpoint
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch add-partition-info-endpoint
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fda7ee8994befded280105fe97d854450f2bb932
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Sep 20 14:37:08 2018 +0200
Add /_partition/:partition/_info endpoint
Add an endpoint that returns the partition size and doc count
---
src/chttpd/src/chttpd_db.erl | 4 ++
src/couch/src/couch_bt_engine.erl | 8 ++-
src/couch/src/couch_db.erl | 11 ++-
src/couch/src/couch_db_engine.erl | 24 +++++++
src/fabric/src/fabric.erl | 9 ++-
src/fabric/src/fabric_db_partition_info.erl | 100 ++++++++++++++++++++++++++++
src/fabric/src/fabric_rpc.erl | 5 +-
7 files changed, 156 insertions(+), 5 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 9dab4b5..c06d224 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -252,6 +252,10 @@ handle_view_cleanup_req(Req, Db) ->
send_json(Req, 202, {[{ok, true}]}).
+handle_partition_req(#httpd{method='GET',path_parts=[DbName, <<"_partition">>, Partition, <<"_info">>]}=Req, _Db) ->
+ {ok, PartitionInfo} = fabric:get_partition_info(DbName, Partition),
+ send_json(Req, {PartitionInfo});
+
handle_partition_req(#httpd{
path_parts=[DbName, <<"_partition">>, Partition, _Design, Name, <<"_",_/binary>> = Action | _Rest]
}=Req, Db) ->
diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index e24d55e..9cd4f3f 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -299,8 +299,12 @@ get_partition_info(#st{} = St, Partition) ->
{partition, Partition},
{doc_count, DocCount},
{doc_del_count, DocDelCount},
- {active, SizeInfo#size_info.active},
- {external, SizeInfo#size_info.external}
+ {size,
+ [
+ {active, SizeInfo#size_info.active},
+ {external, SizeInfo#size_info.external}
+ ]
+ }
].
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index a5c71d7..a2a7534 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -113,7 +113,9 @@
validate_dbname/1,
make_doc/5,
- new_revid/1
+ new_revid/1,
+
+ get_partition_info/2
]).
@@ -475,6 +477,13 @@ get_db_info(Db) ->
],
{ok, InfoList}.
+get_partition_info(#db{} = Db, Partition) when is_binary(Partition) ->
+ Sizes = couch_db_engine:get_partition_info(Db, Partition),
+ {ok, Sizes};
+get_partition_info(_Db, _Partition) ->
+ throw({badrequest, <<"`partition` is not valid">>}).
+
+
get_design_docs(#db{name = <<"shards/", _:18/binary, DbName/binary>>}) ->
{_, Ref} = spawn_monitor(fun() -> exit(fabric:design_docs(DbName)) end),
receive {'DOWN', Ref, _, _, Response} ->
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index 9b28812..910c34e 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -42,6 +42,12 @@
-type purge_info() :: [{docid(), revs()}].
-type epochs() :: [{Node::atom(), UpdateSeq::non_neg_integer()}].
-type size_info() :: [{Name::atom(), Size::non_neg_integer()}].
+-type partition_info() :: [
+ {Partition::atom(), Partition::binary()} |
+ {DocCount::atom(), DocCount::non_neg_integer()} |
+ {DocDelCount::atom(), DocDelCount::non_neg_integer()} |
+ {Size::atom(), size_info()}
+].
-type write_stream_options() :: [
{buffer_size, Size::pos_integer()} |
@@ -254,6 +260,18 @@
-callback get_size_info(DbHandle::db_handle()) -> SizeInfo::size_info().
+% This returns the information for the given partition.
+% It should just be a list of {Name::atom(), Size::non_neg_integer()}
+% It returns the partition name, doc count, deleted doc count and two sizes:
+%
+% active - Theoretical minimum number of bytes to store this partition on disk
+%
+% external - Number of bytes that would be required to represent the
+% contents of this partition outside of the database
+-callback get_partition_info(DbHandle::db_handle(), Partition::binary()) ->
+ partition_info().
+
+
% The current update sequence of the database. The update
% sequence should be incrememnted for every revision added to
% the database.
@@ -627,6 +645,7 @@
get_prop/2,
get_prop/3,
get_size_info/1,
+ get_partition_info/2,
get_update_seq/1,
get_uuid/1,
@@ -803,6 +822,11 @@ get_size_info(#db{} = Db) ->
Engine:get_size_info(EngineState).
+get_partition_info(#db{} = Db, Partition) ->
+ #db{engine = {Engine, EngineState}} = Db,
+ Engine:get_partition_info(EngineState, Partition).
+
+
get_update_seq(#db{} = Db) ->
#db{engine = {Engine, EngineState}} = Db,
Engine:get_update_seq(EngineState).
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index f5c7937..c6b7c5b 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -21,7 +21,7 @@
delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3,
set_security/2, set_security/3, get_revs_limit/1, get_security/1,
get_security/2, get_all_security/1, get_all_security/2,
- compact/1, compact/2]).
+ compact/1, compact/2, get_partition_info/2]).
% Documents
-export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
@@ -84,6 +84,13 @@ all_dbs(Prefix) when is_list(Prefix) ->
get_db_info(DbName) ->
fabric_db_info:go(dbname(DbName)).
+%% @doc returns the size of a given partition
+-spec get_partition_info(dbname(), Partition::binary()) ->
+ {ok, {partition_info, non_neg_integer()}}.
+get_partition_info(DbName, Partition) ->
+ fabric_db_partition_info:go(dbname(DbName), Partition).
+
+
%% @doc the number of docs in a database
-spec get_doc_count(dbname()) ->
{ok, non_neg_integer()} |
diff --git a/src/fabric/src/fabric_db_partition_info.erl b/src/fabric/src/fabric_db_partition_info.erl
new file mode 100644
index 0000000..a742b06
--- /dev/null
+++ b/src/fabric/src/fabric_db_partition_info.erl
@@ -0,0 +1,100 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_db_partition_info).
+
+-export([go/2]).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("mem3/include/mem3.hrl").
+
+go(DbName, Partition) ->
+ Shards = mem3:shards(DbName, <<Partition/binary, ":foo">>),
+ Workers = fabric_util:submit_jobs(Shards, get_partition_info, [Partition]),
+ RexiMon = fabric_util:create_monitors(Shards),
+ Fun = fun handle_message/3,
+ Acc0 = {fabric_dict:init(Workers, nil), []},
+ try
+ case fabric_util:recv(Workers, #shard.ref, Fun, Acc0) of
+ {ok, Acc} -> {ok, Acc};
+ {timeout, {WorkersDict, _}} ->
+ DefunctWorkers = fabric_util:remove_done_workers(
+ WorkersDict,
+ nil
+ ),
+ fabric_util:log_timeout(
+ DefunctWorkers,
+ "get_partition_info"
+ ),
+ {error, timeout};
+ {error, Error} -> throw(Error)
+ end
+ after
+ rexi_monitor:stop(RexiMon)
+ end.
+
+handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Acc}) ->
+ case fabric_util:remove_down_workers(Counters, NodeRef) of
+ {ok, NewCounters} ->
+ {ok, {NewCounters, Acc}};
+ error ->
+ {error, {nodedown, <<"progress not possible">>}}
+ end;
+
+handle_message({rexi_EXIT, Reason}, Shard, {Counters, Acc}) ->
+ NewCounters = fabric_dict:erase(Shard, Counters),
+ case fabric_view:is_progress_possible(NewCounters) of
+ true ->
+ {ok, {NewCounters, Acc}};
+ false ->
+ {error, Reason}
+ end;
+
+handle_message({ok, Sizes}, #shard{dbname=Name} = Shard, {Counters, Acc}) ->
+ Acc2 = [Sizes | Acc],
+ Counters1 = fabric_dict:erase(Shard, Counters),
+ case fabric_dict:size(Counters1) =:= 0 of
+ true ->
+ [FirstInfo | RestInfos] = Acc2,
+ PartitionInfo = get_max_partition_size(FirstInfo, RestInfos),
+ {stop,
+ [{db_name, Name} | format_partition(PartitionInfo)]
+ };
+ false ->
+ {ok, {Counters1, Acc2}}
+ end;
+
+handle_message(_, _, Acc) ->
+ {ok, Acc}.
+
+get_max_partition_size(Max, []) ->
+ Max;
+get_max_partition_size(MaxInfo, [NextInfo | Rest]) ->
+ {size, MaxSize} = lists:keyfind(size, 1, MaxInfo),
+ {size, NextSize} = lists:keyfind(size, 1, NextInfo),
+
+ {external, MaxExtSize} = lists:keyfind(external, 1, MaxSize),
+ {external, NextExtSize} = lists:keyfind(external, 1, NextSize),
+ case NextExtSize > MaxExtSize of
+ true ->
+ get_max_partition_size(NextInfo, Rest);
+ false ->
+ get_max_partition_size(MaxInfo, Rest)
+ end.
+
+
+% for JS to work nicely we need to convert the size list
+% to a jiffy object
+format_partition(PartitionInfo) ->
+ {value, {size, Size}, PartitionInfo1} = lists:keytake(size, 1, PartitionInfo),
+ [{size, {Size}} | PartitionInfo1].
+
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index e538a9d..20bca0e 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -18,7 +18,7 @@
-export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]).
-export([create_db/1, create_db/2, delete_db/1, reset_validation_funs/1,
set_security/3, set_revs_limit/3, create_shard_db_doc/2,
- delete_shard_db_doc/2]).
+ delete_shard_db_doc/2, get_partition_info/2]).
-export([get_all_security/2, open_shard/2]).
-export([compact/1, compact/2]).
@@ -167,6 +167,9 @@ get_db_info(DbName) ->
get_db_info(DbName, DbOptions) ->
with_db(DbName, DbOptions, {couch_db, get_db_info, []}).
+get_partition_info(DbName, Partition) ->
+ with_db(DbName, [], {couch_db, get_partition_info, [Partition]}).
+
%% equiv get_doc_count(DbName, [])
get_doc_count(DbName) ->
get_doc_count(DbName, []).