You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2018/08/29 14:55:31 UTC
[couchdb] branch user-partitioned-dbs-6 updated (a416244 -> fd9c91c)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
omit a416244 add /_partition/partition/designdoc/ endpoints
omit df7efb5 Add restrictions to partitioned views
omit eab120d Add test for all_docs partition optimisation
omit 6630128 optimize _all_docs requests that are bounded within a single partition
omit 183e059 Implement _all_docs and _find support
omit 0ff8646 mem3 tests for partitioned databases
omit 4369b7d implement partitioned views
omit f51279a validate new partition arguments
omit 4035da9 Select only the shard containing the partition, if specified
omit 3599a09 Allow partitioned option in mrviews
omit 09105e2 expose partitioned flag in GET /dbname info
omit b77e082 Enforce partition:id format in doc ids
omit 6b5f7a7 map documents to shards by their partition
new 77fd776 map documents to shards by their partition
new e55bd2b Enforce partition:id format in doc ids
new 6586fe8 expose partitioned flag in GET /dbname info
new 5f30a24 Allow partitioned option in mrviews
new 418ceab Select only the shard containing the partition, if specified
new 386e16b validate new partition arguments
new a8141e2 implement partitioned views
new 2dff101 mem3 tests for partitioned databases
new b6fd946 Implement _all_docs and _find support
new 11f8ca2 optimize _all_docs requests that are bounded within a single partition
new e80eb7e Add test for all_docs partition optimisation
new 81ecd0f Add restrictions to partitioned views
new fd9c91c Add /_partition/$partition/... endpoints
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a416244)
\
N -- N -- N refs/heads/user-partitioned-dbs-6 (fd9c91c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd_db.erl | 23 +++++------------------
src/couch_mrview/src/couch_mrview_http.erl | 6 ++++--
src/fabric/src/fabric.erl | 23 ++++++++---------------
src/mem3/src/mem3.erl | 14 +++++---------
4 files changed, 22 insertions(+), 44 deletions(-)
[couchdb] 01/13: map documents to shards by their partition
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 77fd776f6310d1c60f4865af8820beaeb92f5c33
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Aug 1 18:45:52 2018 +0100
map documents to shards by their partition
---
src/mem3/src/mem3.erl | 37 ++++++++++++++++++++++++++++++++-----
src/mem3/src/mem3_shards.erl | 20 +++++++++++++++++---
src/mem3/src/mem3_util.erl | 23 +++++++++++++++++++++++
3 files changed, 72 insertions(+), 8 deletions(-)
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index 0e5eabf..531f35c 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -13,7 +13,7 @@
-module(mem3).
-export([start/0, stop/0, restart/0, nodes/0, node_info/2, shards/1, shards/2,
- choose_shards/2, n/1, n/2, dbname/1, ushards/1]).
+ choose_shards/2, n/1, n/2, dbname/1, ushards/1, ushards/2]).
-export([get_shard/3, local_shards/1, shard_suffix/1, fold_shards/2]).
-export([sync_security/0, sync_security/1]).
-export([compare_nodelists/0, compare_shards/1]).
@@ -21,6 +21,7 @@
-export([live_shards/2]).
-export([belongs/2, owner/3]).
-export([get_placement/1]).
+-export([is_partitioned/1]).
%% For mem3 use only.
-export([name/1, node/1, range/1, engine/1]).
@@ -68,7 +69,7 @@ compare_shards(DbName) ->
-spec n(DbName::iodata()) -> integer().
n(DbName) ->
- n(DbName, <<"foo">>).
+ n(DbName, <<"_design/foo">>).
n(DbName, DocId) ->
length(mem3:shards(DbName, DocId)).
@@ -133,6 +134,12 @@ ushards(DbName) ->
Shards = ushards(DbName, live_shards(DbName, Nodes, [ordered]), ZoneMap),
mem3_util:downcast(Shards).
+-spec ushards(DbName::iodata(), DocId::binary()) -> [#shard{}].
+ushards(DbName, DocId) ->
+ Shards = shards_int(DbName, DocId, [ordered]),
+ Shard = hd(Shards),
+ mem3_util:downcast([Shard]).
+
ushards(DbName, Shards0, ZoneMap) ->
{L,S,D} = group_by_proximity(Shards0, ZoneMap),
% Prefer shards in the local zone over shards in a different zone,
@@ -229,13 +236,14 @@ dbname(_) ->
erlang:error(badarg).
%% @doc Determine if DocId belongs in shard (identified by record or filename)
-belongs(#shard{}=Shard, DocId) when is_binary(DocId) ->
+%% NOTE: only supported for design documents
+belongs(#shard{}=Shard, <<"_design/", _/binary>> = DocId) ->
[Begin, End] = range(Shard),
belongs(Begin, End, DocId);
-belongs(<<"shards/", _/binary>> = ShardName, DocId) when is_binary(DocId) ->
+belongs(<<"shards/", _/binary>> = ShardName, <<"_design/", _/binary>> = DocId) ->
[Begin, End] = range(ShardName),
belongs(Begin, End, DocId);
-belongs(DbName, DocId) when is_binary(DbName), is_binary(DocId) ->
+belongs(DbName, <<"_design/", _/binary>>) when is_binary(DbName) ->
true.
belongs(Begin, End, DocId) ->
@@ -331,6 +339,25 @@ engine(Opts) when is_list(Opts) ->
[]
end.
+is_partitioned(DbName0) when is_binary(DbName0) ->
+ DbName = dbname(DbName0),
+ try
+ is_partitioned(mem3:shards(DbName))
+ catch
+ error:database_does_not_exist ->
+ false
+ end;
+
+is_partitioned(Shards) when is_list(Shards) ->
+ lists:all(fun is_partitioned/1, Shards);
+
+is_partitioned(#shard{opts=Opts}) ->
+ couch_util:get_value(partitioned, Opts) == true;
+
+is_partitioned(#ordered_shard{opts=Opts}) ->
+ couch_util:get_value(partitioned, Opts) == true.
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl
index da3b69a..1cd7be6 100644
--- a/src/mem3/src/mem3_shards.erl
+++ b/src/mem3/src/mem3_shards.erl
@@ -66,8 +66,22 @@ for_db(DbName, Options) ->
for_docid(DbName, DocId) ->
for_docid(DbName, DocId, []).
+%% This function performs one or two lookups now as it is not known
+%% ahead of time if the database is partitioned We first ask for the
+%% shards as if the database is not partitioned and then test the
+%% returned shards for a counter-indication that it was. If so, we
+%% run the function again with the docid hash option enabled.
for_docid(DbName, DocId, Options) ->
- HashKey = mem3_util:hash(DocId),
+ Shards = for_docid(DbName, DocId, Options, [{partitioned, false}]),
+ case mem3:is_partitioned(Shards) of
+ true ->
+ for_docid(DbName, DocId, Options, [{partitioned, true}]);
+ false ->
+ Shards
+ end.
+
+for_docid(DbName, DocId, Options, HashOptions) ->
+ HashKey = mem3_util:docid_hash(DocId, HashOptions),
ShardHead = #shard{
dbname = DbName,
range = ['$1', '$2'],
@@ -397,7 +411,8 @@ load_shards_from_db(ShardDb, DbName) ->
load_shards_from_disk(DbName, DocId)->
Shards = load_shards_from_disk(DbName),
- HashKey = mem3_util:hash(DocId),
+ Options = [{partitioned, mem3:is_partitioned(Shards)}],
+ HashKey = mem3_util:docid_hash(DocId, Options),
[S || S <- Shards, in_range(S, HashKey)].
in_range(Shard, HashKey) ->
@@ -521,7 +536,6 @@ filter_shards_by_name(Name, Matches, [#shard{name=Name}=S|Ss]) ->
filter_shards_by_name(Name, Matches, [_|Ss]) ->
filter_shards_by_name(Name, Matches, Ss).
-
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index e08d375..9620e98 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -16,6 +16,7 @@
n_val/2, to_atom/1, to_integer/1, write_db_doc/1, delete_db_doc/1,
shard_info/1, ensure_exists/1, open_db_doc/1]).
-export([is_deleted/1, rotate_list/2]).
+-export([docid_hash/1, docid_hash/2]).
%% do not use outside mem3.
-export([build_ordered_shards/2, downcast/1]).
@@ -34,6 +35,28 @@ hash(Item) when is_binary(Item) ->
hash(Item) ->
erlang:crc32(term_to_binary(Item)).
+
+docid_hash(DocId) when is_binary(DocId) ->
+ docid_hash(DocId, []).
+
+docid_hash(<<"_design/", _/binary>> = DocId, _Options) ->
+ erlang:crc32(DocId); % design docs are never placed by partition
+
+docid_hash(DocId, []) when is_binary(DocId) ->
+ docid_hash(DocId, [{partitioned, false}]);
+
+docid_hash(DocId, [{partitioned, false}]) when is_binary(DocId) ->
+ erlang:crc32(DocId);
+
+docid_hash(DocId, [{partitioned, true}]) when is_binary(DocId) ->
+ case binary:split(DocId, <<":">>) of
+ [Partition, _Rest] ->
+ erlang:crc32(Partition);
+ _ ->
+ throw({illegal_docid, <<"doc id must be of form partition:id">>})
+ end.
+
+
name_shard(Shard) ->
name_shard(Shard, "").
[couchdb] 07/13: implement partitioned views
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a8141e212583d802bb84e5b54925b14c08bc78f1
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Aug 7 15:44:33 2018 +0100
implement partitioned views
Co-authored-by: Robert Newson <rn...@apache.org>
Co-authored-by: Paul J. Davis <pa...@gmail.com>
---
src/couch/src/couch_btree.erl | 14 +++++++
src/couch/src/couch_ejson_compare.erl | 4 ++
src/couch_mrview/src/couch_mrview.erl | 2 +
src/couch_mrview/src/couch_mrview_updater.erl | 14 ++++++-
src/couch_mrview/src/couch_mrview_util.erl | 53 ++++++++++++++++++++++++---
src/fabric/src/fabric_view.erl | 18 +++++++--
6 files changed, 95 insertions(+), 10 deletions(-)
diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl
index ea224b1..d11d7e6 100644
--- a/src/couch/src/couch_btree.erl
+++ b/src/couch/src/couch_btree.erl
@@ -133,6 +133,20 @@ make_group_fun(Bt, exact) ->
end;
make_group_fun(Bt, GroupLevel) when is_integer(GroupLevel), GroupLevel > 0 ->
fun
+ ({{p, _Partition, Key1}, _}, {{p, _Partition, Key2}, _}) ->
+ SL1 = lists:sublist(Key1, GroupLevel),
+ SL2 = lists:sublist(Key2, GroupLevel),
+ case less(Bt, {SL1, nil}, {SL2, nil}) of
+ false ->
+ case less(Bt, {SL2, nil}, {SL1, nil}) of
+ false ->
+ true;
+ _ ->
+ false
+ end;
+ _ ->
+ false
+ end;
({[_|_] = Key1, _}, {[_|_] = Key2, _}) ->
SL1 = lists:sublist(Key1, GroupLevel),
SL2 = lists:sublist(Key2, GroupLevel),
diff --git a/src/couch/src/couch_ejson_compare.erl b/src/couch/src/couch_ejson_compare.erl
index 81adbb8..ca36c86 100644
--- a/src/couch/src/couch_ejson_compare.erl
+++ b/src/couch/src/couch_ejson_compare.erl
@@ -22,6 +22,10 @@ init() ->
Dir = code:priv_dir(couch),
ok = erlang:load_nif(filename:join(Dir, ?MODULE), NumScheds).
+% partitioned row comparison
+less({p, PA, A}, {p, PB, B}) ->
+ less([PA, A], [PB, B]);
+
less(A, B) ->
try
less_nif(A, B)
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index db467f0..09945f5 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -614,6 +614,8 @@ red_fold(Db, {NthRed, _Lang, View}=RedView, Args, Callback, UAcc) ->
end, Acc, OptList),
finish_fold(Acc2, []).
+red_fold({p, _Partition, Key}, Red, Acc) ->
+ red_fold(Key, Red, Acc);
red_fold(_Key, _Red, #mracc{skip=N}=Acc) when N > 0 ->
{ok, Acc#mracc{skip=N-1, last_go=ok}};
red_fold(Key, Red, #mracc{meta_sent=false}=Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 214f487..2b69eee 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -311,9 +311,11 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
#mrst{
id_btree=IdBtree,
log_btree=LogBtree,
- first_build=FirstBuild
+ first_build=FirstBuild,
+ design_opts=DesignOpts
} = State,
+ Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, false),
Revs = dict:from_list(dict:fetch_keys(Log0)),
Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
@@ -328,8 +330,9 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
_ -> update_log(LogBtree, Log, Revs, Seqs, FirstBuild)
end,
- UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
+ UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs0, SKVs}}) ->
#mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
+ KVs = if Partitioned -> inject_partition(KVs0); true -> KVs0 end,
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
@@ -378,6 +381,13 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
log_btree=LogBtree2
}.
+inject_partition(KVs) ->
+ [{{{p, partition(DocId), Key}, DocId}, Value} || {{Key, DocId}, Value} <- KVs].
+
+partition(DocId) ->
+ [Partition, _Rest] = binary:split(DocId, <<":">>),
+ Partition.
+
update_id_btree(Btree, DocIdKeys, true) ->
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
couch_btree:query_modify(Btree, [], ToAdd, []);
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 592bfb5..574aac7 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -38,6 +38,9 @@
-define(MOD, couch_mrview_index).
-define(GET_VIEW_RETRY_COUNT, 1).
-define(GET_VIEW_RETRY_DELAY, 50).
+-define(LOWEST_KEY, null).
+-define(HIGHEST_KEY, {[{<<239, 191, 176>>, null}]}). % is {"\ufff0": null}
+
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -226,11 +229,12 @@ view_sig(_Db, State, View, Args0) ->
PurgeSeq = View#mrview.purge_seq,
SeqIndexed = View#mrview.seq_indexed,
KeySeqIndexed = View#mrview.keyseq_indexed,
+ Partitioned = get_extra(Args0, partitioned, false),
Args = Args0#mrargs{
preflight_fun=undefined,
extra=[]
},
- Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args),
+ Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Partitioned, Args),
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Term))).
view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false) ->
@@ -238,10 +242,12 @@ view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false) ->
view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed) ->
{BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed}.
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false, Args) ->
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false, false, Args) ->
{BaseSig, UpdateSeq, PurgeSeq, Args};
-view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) ->
- {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}.
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, false, Args) ->
+ {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args};
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Partitioned, Args) ->
+ {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Partitioned, Args}.
init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
@@ -588,7 +594,12 @@ validate_args(Args) ->
mrverror(<<"`partition` parameter is not supported in this view.">>)
end,
- Args#mrargs{
+ Args1 = case get_extra(Args, partitioned, false) of
+ true -> apply_partition(Args);
+ false -> Args
+ end,
+
+ Args1#mrargs{
start_key_docid=SKDocId,
end_key_docid=EKDocId,
group_level=GroupLevel
@@ -606,6 +617,38 @@ determine_group_level(#mrargs{group=true, group_level=undefined}) ->
determine_group_level(#mrargs{group_level=GroupLevel}) ->
GroupLevel.
+apply_partition(#mrargs{} = Args0) ->
+ Partition = get_extra(Args0, partition),
+ apply_partition(Partition, Args0).
+
+apply_partition(_Partition, #mrargs{keys=[{p, _, _} | _]} = Args) ->
+ Args; % already applied
+
+apply_partition(Partition, #mrargs{keys=Keys} = Args) when Keys /= undefined ->
+ Args#mrargs{keys=[{p, Partition, K} || K <- Keys]};
+
+apply_partition(_Partition, #mrargs{start_key={p, _, _}, end_key={p, _, _}} = Args) ->
+ Args; % already applied.
+
+apply_partition(Partition, Args) ->
+ #mrargs{
+ direction = Dir,
+ start_key = StartKey,
+ end_key = EndKey
+ } = Args,
+
+ {DefSK, DefEK} = case Dir of
+ fwd -> {?LOWEST_KEY, ?HIGHEST_KEY};
+ rev -> {?HIGHEST_KEY, ?LOWEST_KEY}
+ end,
+
+ SK0 = if StartKey /= undefined -> StartKey; true -> DefSK end,
+ EK0 = if EndKey /= undefined -> EndKey; true -> DefEK end,
+
+ Args#mrargs{
+ start_key = {p, Partition, SK0},
+ end_key = {p, Partition, EK0}
+ }.
check_range(#mrargs{start_key=undefined}, _Cmp) ->
ok;
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index b4b8a8c..844b44d 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -119,8 +119,10 @@ maybe_send_row(State) ->
counters = Counters,
skip = Skip,
limit = Limit,
- user_acc = AccIn
+ user_acc = AccIn,
+ query_args = QueryArgs
} = State,
+ Partitioned = couch_mrview_util:get_extra(QueryArgs, partitioned, false),
case fabric_dict:any(0, Counters) of
true ->
{ok, State};
@@ -128,8 +130,14 @@ maybe_send_row(State) ->
try get_next_row(State) of
{_, NewState} when Skip > 0 ->
maybe_send_row(NewState#collector{skip=Skip-1});
- {Row, NewState} ->
- case Callback(transform_row(possibly_embed_doc(NewState,Row)), AccIn) of
+ {Row0, NewState} ->
+ Row1 = possibly_embed_doc(NewState, Row0),
+ Row2 = if
+ Partitioned -> detach_partition(Row1);
+ true -> Row1
+ end,
+ Row3 = transform_row(Row2),
+ case Callback(Row3, AccIn) of
{stop, Acc} ->
{stop, NewState#collector{user_acc=Acc, limit=Limit-1}};
{ok, Acc} ->
@@ -194,6 +202,10 @@ possibly_embed_doc(#collector{db_name=DbName, query_args=Args},
_ -> Row
end.
+detach_partition(#view_row{key={p, _Partition, Key}} = Row) ->
+ Row#view_row{key = Key};
+detach_partition(#view_row{} = Row) ->
+ Row.
keydict(undefined) ->
undefined;
[couchdb] 11/13: Add test for all_docs partition optimisation
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e80eb7e48ef586b4cf4b8c763ddb4ac0eb90504d
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Aug 14 15:43:01 2018 +0200
Add test for all_docs partition optimisation
Adds tests to validate the all_docs optimisations works for partitions
---
src/fabric/src/fabric_view_all_docs.erl | 86 +++++++++++++++++++++++++++++++++
1 file changed, 86 insertions(+)
diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl
index 83c3790..b12bcde 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -316,3 +316,89 @@ cancel_read_pids(Pids) ->
{empty, _} ->
ok
end.
+
+-ifdef(TEST).
+
+-include_lib("eunit/include/eunit.hrl").
+
+ shards_for_partition_gets_partitioned_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ start_key = <<"pk:id">>,
+ end_key = <<"pk:idZ">>,
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>, <<"pk:foo">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_no_partition_gets_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ start_key = <<"pk:id">>,
+ end_key = <<"pk:idZ">>,
+ extra = [{partitioned, false}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_different_partitions_gets_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ start_key = <<"pk1:id">>,
+ end_key = <<"pk2:idZ">>,
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_no_startkey_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ end_key = <<"pk:idZ">>,
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_no_endkey_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ start_key = <<"pk:idZ">>,
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_no_keys_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+ shards_for_non_binary_keys_all_shards_test() ->
+ DbName = <<"db">>,
+ Args = #mrargs{
+ start_key = null,
+ end_key = null,
+ extra = [{partitioned, true}]
+ },
+ meck:expect(mem3, shards, fun(<<"db">>) -> [] end),
+ shards(DbName, Args),
+ meck:validate(mem3),
+ meck:unload(mem3).
+
+-endif.
[couchdb] 10/13: optimize _all_docs requests that are bounded
within a single partition
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 11f8ca21d3676d2e7eef48d8f483deeff917f5d0
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Aug 13 22:34:59 2018 +0100
optimize _all_docs requests that are bounded within a single partition
---
src/fabric/src/fabric_view_all_docs.erl | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl
index d515ab8..83c3790 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -21,7 +21,7 @@
-include_lib("couch_mrview/include/couch_mrview.hrl").
go(DbName, Options, #mrargs{keys=undefined} = QueryArgs, Callback, Acc) ->
- Shards = mem3:shards(DbName),
+ Shards = shards(DbName, QueryArgs),
Workers0 = fabric_util:submit_jobs(
Shards, fabric_rpc, all_docs, [Options, QueryArgs]),
RexiMon = fabric_util:create_monitors(Workers0),
@@ -136,6 +136,28 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) ->
{ok, Resp}
end.
+shards(DbName, Args) ->
+ case couch_mrview_util:get_extra(Args, partitioned) of
+ true ->
+ StartKey = partition(Args#mrargs.start_key),
+ EndKey = partition(Args#mrargs.end_key),
+ case {StartKey, EndKey} of
+ {Same, Same} when Same =/= undefined ->
+ mem3:shards(DbName, <<Same/binary, ":foo">>);
+ {_, _} ->
+ mem3:shards(DbName)
+ end;
+ _ ->
+ mem3:shards(DbName)
+ end.
+
+partition(undefined) ->
+ undefined;
+partition(null) ->
+ null;
+partition(Key) when is_binary(Key) ->
+ hd(binary:split(Key, <<":">>)).
+
handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, State) ->
fabric_view:check_down_shards(State, NodeRef);
[couchdb] 04/13: Allow partitioned option in mrviews
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5f30a24d53a68ee28bf7e58d0e4a48cc5bc8e89b
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Aug 6 18:18:33 2018 +0100
Allow partitioned option in mrviews
Default to database's partitioned setting if not present in ddoc.
---
src/chttpd/src/chttpd_db.erl | 4 ++--
src/couch_mrview/src/couch_mrview.erl | 5 +++++
src/couch_mrview/src/couch_mrview_http.erl | 9 +++++++--
src/couch_mrview/src/couch_mrview_index.erl | 4 +++-
src/couch_mrview/src/couch_mrview_util.erl | 14 ++++++++++++++
src/fabric/src/fabric.erl | 1 +
6 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 63d2907..ddcad97 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -1607,8 +1607,8 @@ set_namespace(<<"_local_docs">>, Args) ->
set_namespace(<<"_local">>, Args);
set_namespace(<<"_design_docs">>, Args) ->
set_namespace(<<"_design">>, Args);
-set_namespace(NS, #mrargs{extra = Extra} = Args) ->
- Args#mrargs{extra = [{namespace, NS} | Extra]}.
+set_namespace(NS, #mrargs{} = Args) ->
+ couch_mrview_util:set_extra(Args, namespace, NS).
%% /db/_bulk_get stuff
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 533dd2d..db467f0 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -57,6 +57,9 @@ validate_ddoc_fields(DDoc) ->
[{<<"language">>, string}],
[{<<"lists">>, object}, {any, [object, string]}],
[{<<"options">>, object}],
+ [{<<"options">>, object}, {<<"include_design">>, boolean}],
+ [{<<"options">>, object}, {<<"partitioned">>, boolean}],
+ [{<<"options">>, object}, {<<"local_seq">>, boolean}],
[{<<"rewrites">>, [string, array]}],
[{<<"shows">>, object}, {any, [object, string]}],
[{<<"updates">>, object}, {any, [object, string]}],
@@ -133,6 +136,8 @@ validate_ddoc_field(Value, array) when is_list(Value) ->
ok;
validate_ddoc_field({Value}, object) when is_list(Value) ->
ok;
+validate_ddoc_field(Value, boolean) when is_boolean(Value) ->
+ ok;
validate_ddoc_field({Props}, {any, Type}) ->
validate_ddoc_field1(Props, Type);
validate_ddoc_field({Props}, {Key, Type}) ->
diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl
index 004caef..dfdb644 100644
--- a/src/couch_mrview/src/couch_mrview_http.erl
+++ b/src/couch_mrview/src/couch_mrview_http.erl
@@ -461,8 +461,10 @@ row_to_json(Id0, Row) ->
?JSON_ENCODE(Obj).
-parse_params(#httpd{}=Req, Keys) ->
- parse_params(chttpd:qs(Req), Keys);
+parse_params(#httpd{path_parts=[DbName | _]}=Req, Keys) ->
+ Args = parse_params(chttpd:qs(Req), Keys),
+ Partitioned = mem3:is_partitioned(DbName),
+ couch_mrview_util:set_extra(Args, partitioned, Partitioned);
parse_params(Props, Keys) ->
Args = #mrargs{},
parse_params(Props, Keys, Args).
@@ -582,6 +584,9 @@ parse_param(Key, Val, Args, IsDecoded) ->
Args#mrargs{callback=couch_util:to_binary(Val)};
"sorted" ->
Args#mrargs{sorted=parse_boolean(Val)};
+ "partition" ->
+ Partition = couch_util:to_binary(Val),
+ couch_mrview_util:set_extra(Args, partition, Partition);
_ ->
BKey = couch_util:to_binary(Key),
BVal = couch_util:to_binary(Val),
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 5d285d6..2d462d9 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -38,10 +38,12 @@ get(update_options, #mrst{design_opts = Opts}) ->
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
KeySeqIndexed = couch_util:get_value(<<"keyseq_indexed">>, Opts, false),
+ Partitioned = couch_util:get_value(<<"partitioned">>, Opts, false),
if IncDesign -> [include_design]; true -> [] end
++ if LocalSeq -> [local_seq]; true -> [] end
++ if KeySeqIndexed -> [keyseq_indexed]; true -> [] end
- ++ if SeqIndexed -> [seq_indexed]; true -> [] end;
+ ++ if SeqIndexed -> [seq_indexed]; true -> [] end
+ ++ if Partitioned -> [partitioned]; true -> [] end;
get(fd, #mrst{fd = Fd}) ->
Fd;
get(language, #mrst{language = Language}) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 120a9b8..a63701f 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -30,6 +30,7 @@
-export([extract_view/4, extract_view_reduce/1]).
-export([get_view_keys/1, get_view_queries/1]).
-export([set_view_type/3]).
+-export([set_extra/3, get_extra/2, get_extra/3]).
-export([changes_key_opts/2]).
-export([fold_changes/4]).
-export([to_key_seq/1]).
@@ -178,6 +179,19 @@ set_view_type(Args, ViewName, [View | Rest]) ->
end.
+set_extra(#mrargs{} = Args, Key, Value) ->
+ Extra0 = Args#mrargs.extra,
+ Extra1 = lists:ukeysort(1, [{Key, Value} | Extra0]),
+ Args#mrargs{extra = Extra1}.
+
+
+get_extra(#mrargs{} = Args, Key) ->
+ couch_util:get_value(Key, Args#mrargs.extra).
+
+get_extra(#mrargs{} = Args, Key, Default) ->
+ couch_util:get_value(Key, Args#mrargs.extra, Default).
+
+
extract_view(_Lang, _Args, _ViewName, []) ->
throw({not_found, missing_named_view});
extract_view(Lang, #mrargs{view_type=map}=Args, Name, [View | Rest]) ->
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index f5c7937..0295597 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -355,6 +355,7 @@ query_view(DbName, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) ->
end,
{ok, #mrst{views=Views, language=Lang}} =
couch_mrview_util:ddoc_to_mrst(Db, DDoc),
+
QueryArgs1 = couch_mrview_util:set_view_type(QueryArgs0, View, Views),
QueryArgs2 = couch_mrview_util:validate_args(QueryArgs1),
VInfo = couch_mrview_util:extract_view(Lang, QueryArgs2, View, Views),
[couchdb] 06/13: validate new partition arguments
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 386e16be5d1accb787bcc5487324ca341b1ee393
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Aug 6 19:21:49 2018 +0100
validate new partition arguments
---
src/couch_mrview/src/couch_mrview_util.erl | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index a63701f..592bfb5 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -577,6 +577,17 @@ validate_args(Args) ->
_ -> mrverror(<<"Invalid value for `sorted`.">>)
end,
+ case {get_extra(Args, partitioned, false), get_extra(Args, partition)} of
+ {true, undefined} ->
+ mrverror(<<"`partition` parameter is mandatory for queries to this view.">>);
+ {true, _Partition} ->
+ ok;
+ {false, undefined} ->
+ ok;
+ {false, _Partition} ->
+ mrverror(<<"`partition` parameter is not supported in this view.">>)
+ end,
+
Args#mrargs{
start_key_docid=SKDocId,
end_key_docid=EKDocId,
[couchdb] 12/13: Add restrictions to partitioned views
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 81ecd0ff7466308983c3e99e3dae7a652c97d893
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Aug 22 15:35:51 2018 +0100
Add restrictions to partitioned views
* Block design documents with partitioned option in non-partitioned db
* Prohibit javascript reduces in partitioned:true ddocs
* Prohibit include_docs=true for _view in partitioned db
---
src/chttpd/src/chttpd_view.erl | 3 ++-
src/couch_mrview/src/couch_mrview.erl | 16 ++++++++++++++++
src/couch_mrview/src/couch_mrview_util.erl | 12 +++++++++++-
3 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index 3c05c64..6b95706 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -24,7 +24,7 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
QueryArg = couch_mrview_http:parse_params(Query, undefined,
Args1, [decoded]),
QueryArg1 = couch_mrview_util:set_view_type(QueryArg, ViewName, Views),
- couch_mrview_util:validate_args(QueryArg1)
+ couch_mrview_util:validate_args(QueryArg1, [view])
end, Queries),
Options = [{user_ctx, Req#httpd.user_ctx}],
VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
@@ -42,6 +42,7 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
Args = couch_mrview_http:parse_params(Req, Keys),
+ couch_mrview_util:validate_args(Args, [view]),
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
Options = [{user_ctx, Req#httpd.user_ctx}],
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index f5963e7..7862afb 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -176,6 +176,16 @@ join([H|T], Sep, Acc) ->
validate(DbName, DDoc) ->
ok = validate_ddoc_fields(DDoc#doc.body),
+ DbPartitioned = mem3:is_partitioned(DbName),
+ DDocPartitioned = get_partitioned_opt(DDoc#doc.body, DbPartitioned),
+ if
+ not DbPartitioned andalso DDocPartitioned ->
+ throw({invalid_design_doc,
+ <<"partitioned option cannot be true in a "
+ "non-partitioned database.">>});
+ true ->
+ ok
+ end,
GetName = fun
(#mrview{map_names = [Name | _]}) -> Name;
(#mrview{reduce_funs = [{Name, _} | _]}) -> Name;
@@ -195,6 +205,9 @@ validate(DbName, DDoc) ->
({_RedName, <<"_", _/binary>> = Bad}) ->
Msg = ["`", Bad, "` is not a supported reduce function."],
throw({invalid_design_doc, Msg});
+ ({_RedName, _RedSrc}) when DDocPartitioned ->
+ Msg = <<"Javascript reduces not supported in partitioned view.">>,
+ throw({invalid_design_doc, Msg});
({RedName, RedSrc}) ->
couch_query_servers:try_compile(Proc, reduce, RedName, RedSrc)
end, Reds)
@@ -215,6 +228,9 @@ validate(DbName, DDoc) ->
ok
end.
+get_partitioned_opt({Props}, Default) ->
+ {Options} = couch_util:get_value(<<"options">>, Props, {[]}),
+ couch_util:get_value(<<"partitioned">>, Options, Default).
query_all_docs(Db, Args) ->
query_all_docs(Db, Args, fun default_cb/2, []).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 02e695d..794b694 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -24,7 +24,7 @@
-export([temp_view_to_ddoc/1]).
-export([calculate_external_size/1]).
-export([calculate_active_size/1]).
--export([validate_args/1]).
+-export([validate_args/1, validate_args/2]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
-export([extract_view/4, extract_view_reduce/1]).
@@ -465,6 +465,9 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
validate_args(Args) ->
+ validate_args(Args, []).
+
+validate_args(Args, ValidateOptions) ->
GroupLevel = determine_group_level(Args),
Reduce = Args#mrargs.reduce,
case Reduce == undefined orelse is_boolean(Reduce) of
@@ -607,6 +610,13 @@ validate_args(Args) ->
mrverror(<<"`partition` parameter is not supported in this view.">>)
end,
+ case {Partitioned, Args#mrargs.include_docs, ValidateOptions} of
+ {true, true, [view]} ->
+ mrverror(<<"`include_docs=true` is not supported in this view.">>);
+ {_, _, _} ->
+ ok
+ end,
+
Args1 = case {Style, Partitioned, Partition} of
{all_docs, true, undefined} ->
Args;
[couchdb] 08/13: mem3 tests for partitioned databases
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2dff1017fdbf92edc05e562318bddeac29e949d2
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Aug 7 12:15:14 2018 +0200
mem3 tests for partitioned databases
---
src/mem3/src/mem3.erl | 22 ++++++++++++++
src/mem3/src/mem3_shards.erl | 68 +++++++++++++++++++++++++++++++++++++++++++-
src/mem3/src/mem3_util.erl | 59 +++++++++++++++++++++++++++++++++++++-
3 files changed, 147 insertions(+), 2 deletions(-)
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index 531f35c..bafb227 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -385,4 +385,26 @@ allowed_nodes_test_() ->
]
}]}.
+is_partitioned_false_shards_db_test() ->
+ meck:expect(config, get, fun (_, _, Default) -> Default end),
+ ?assertEqual(is_partitioned(<<"_dbs">>), false),
+ meck:unload().
+
+is_partitioned_false_nodes_db_test() ->
+ meck:expect(config, get, fun (_, _, Default) -> Default end),
+ ?assertEqual(is_partitioned(<<"_nodes">>), false),
+ meck:unload().
+
+is_partitioned_true_partitioned_db_test() ->
+ Shard = #shard{
+ opts = [{partitioned, true}]
+ },
+ ?assertEqual(is_partitioned([Shard]), true).
+
+is_partitioned_false_partitioned_db_test() ->
+ Shard = #shard{
+ opts = []
+ },
+ ?assertEqual(is_partitioned([Shard]), false).
+
-endif.
diff --git a/src/mem3/src/mem3_shards.erl b/src/mem3/src/mem3_shards.erl
index 1cd7be6..cfbc32e 100644
--- a/src/mem3/src/mem3_shards.erl
+++ b/src/mem3/src/mem3_shards.erl
@@ -559,7 +559,9 @@ mem3_shards_test_() ->
t_writer_does_not_delete_other_writers_for_same_shard(),
t_spawn_writer_in_load_shards_from_db(),
t_cache_insert_takes_new_update(),
- t_cache_insert_ignores_stale_update_and_kills_worker()
+ t_cache_insert_ignores_stale_update_and_kills_worker(),
+ t_load_shards_from_disk_returns_correct_shard_for_partition(),
+ t_for_docid_returns_correct_shard_for_partition()
]
}.
@@ -703,6 +705,70 @@ t_cache_insert_ignores_stale_update_and_kills_worker() ->
?assertEqual([], ets:tab2list(?OPENERS))
end).
+t_load_shards_from_disk_returns_correct_shard_for_partition() ->
+ ?_test(begin
+ Shards = [
+ #ordered_shard{
+ name = <<"shards/80000000-9fffffff/db1.1533630706">>,
+ node = 'node1@127.0.0.1',
+ dbname = <<"db1">>,
+ range = [2147483648,2684354559],
+ ref = undefined,
+ order = 1,
+ opts = [{partitioned,true}]
+ }
+ ],
+ DbName = <<"db1">>,
+ DocId = <<"foo:123">>,
+ Doc = #doc{body = {[]}},
+ meck:expect(couch_db, open_doc, 3, {ok, Doc}),
+ meck:expect(couch_db, get_update_seq, 1, 1),
+ meck:expect(mem3_util, build_ordered_shards, 2, Shards),
+ meck:expect(mem3_util, ensure_exists, 1, {ok, <<"shard-name">>}),
+ meck:expect(couch_db, close, 1, ok),
+
+ [Shard] = load_shards_from_disk(DbName, DocId),
+
+ meck:validate(couch_db),
+ meck:validate(mem3_util),
+
+ ShardName = Shard#ordered_shard.name,
+ ?assertEqual(ShardName, <<"shards/80000000-9fffffff/db1.1533630706">>)
+ end).
+
+t_for_docid_returns_correct_shard_for_partition() ->
+ ?_test(begin
+ Shards = [
+ #ordered_shard{
+ name = <<"shards/60000000-7fffffff/db1.1533630706">>,
+ node = 'node1@127.0.0.1',
+ dbname = <<"db1">>,
+ range = [1610612736,2147483647],
+ ref = undefined,
+ order = 1,
+ opts = [{partitioned,true}]
+ },
+ #ordered_shard{
+ name = <<"shards/80000000-9fffffff/db1.1533630706">>,
+ node = 'node1@127.0.0.1',
+ dbname = <<"db1">>,
+ range = [2147483648,2684354559],
+ ref = undefined,
+ order = 1,
+ opts = [{partitioned,true}]
+ }
+ ],
+ DbName = <<"db1">>,
+ DocId = <<"foo:123">>,
+
+ true = ets:insert(?SHARDS, Shards),
+
+ [Shard] = for_docid(DbName, DocId, [ordered]),
+
+ ShardName = Shard#ordered_shard.name,
+ ?assertEqual(ShardName, <<"shards/80000000-9fffffff/db1.1533630706">>)
+ end).
+
mock_state(UpdateSeq) ->
#st{
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index 9620e98..7b8dd16 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -35,7 +35,6 @@ hash(Item) when is_binary(Item) ->
hash(Item) ->
erlang:crc32(term_to_binary(Item)).
-
docid_hash(DocId) when is_binary(DocId) ->
docid_hash(DocId, []).
@@ -294,3 +293,61 @@ downcast(#ordered_shard{}=S) ->
};
downcast(Shards) when is_list(Shards) ->
[downcast(Shard) || Shard <- Shards].
+
+-ifdef(TEST).
+
+-include_lib("eunit/include/eunit.hrl").
+
+docid_hash_design_doc_test() ->
+ Id = <<"_design/ddoc">>,
+ Hash = docid_hash(Id),
+ ?assertEqual(Hash, erlang:crc32(Id)).
+
+docid_hash_doc_partition_false_test() ->
+ Id = <<"partitionkey:docid">>,
+ IdHash = erlang:crc32(Id),
+ Hash = docid_hash(Id),
+ ?assertEqual(Hash, IdHash),
+ Hash = docid_hash(Id, []),
+ ?assertEqual(Hash, IdHash).
+
+docid_hash_doc_partition_true_test() ->
+ Id = <<"partitionkey:doc:id">>,
+ Hash = docid_hash(Id, [{partitioned, true}]),
+ ?assertEqual(Hash, erlang:crc32(<<"partitionkey">>)).
+
+
+add_shards_by_node_adds_partition_prop_test() ->
+ DocProp = [
+ {<<"_id">>, <<"database-name">>},
+ {<<"_rev">>,<<"1-fb8e28457a6e0c49de1848b5e4a28238">>},
+ {<<"shard_suffix">>,".1533550200"},
+ {<<"changelog">>, [[<<"add">>,<<"00000000-1fffffff">>,<<"node1@127.0.0.1">>]]},
+ {<<"by_node">>, {[{<<"node1@127.0.0.1">>, [<<"00000000-1fffffff">>,<<"20000000-3fffffff">>]}]}},
+ {<<"by_range">>, {[{<<"00000000-1fffffff">>,[<<"node1@127.0.0.1">>]}]}},
+ {<<"options">>,{[{partitioned,true}]}}
+ ],
+
+ [ShardRange | _] = build_shards_by_node(<<"database-name">>, DocProp),
+ Opts = ShardRange#shard.opts,
+ Partitioned = lists:keyfind(partitioned, 1, Opts),
+ ?assertEqual(Partitioned, {partitioned, true}).
+
+
+add_shards_by_range_adds_partition_prop_test() ->
+ DocProp = [
+ {<<"_id">>, <<"database-name">>},
+ {<<"_rev">>,<<"1-fb8e28457a6e0c49de1848b5e4a28238">>},
+ {<<"shard_suffix">>,".1533550200"},
+ {<<"changelog">>, [[<<"add">>,<<"00000000-1fffffff">>,<<"node1@127.0.0.1">>]]},
+ {<<"by_node">>, {[{<<"node1@127.0.0.1">>, [<<"00000000-1fffffff">>,<<"20000000-3fffffff">>]}]}},
+ {<<"by_range">>, {[{<<"00000000-1fffffff">>,[<<"node1@127.0.0.1">>]}]}},
+ {<<"options">>,{[{partitioned,true}]}}
+ ],
+
+ [ShardRange | _] = build_shards_by_range(<<"database-name">>, DocProp),
+ Opts = ShardRange#ordered_shard.opts,
+ Partitioned = lists:keyfind(partitioned, 1, Opts),
+ ?assertEqual(Partitioned, {partitioned, true}).
+
+-endif.
[couchdb] 13/13: Add /_partition/$partition/... endpoints
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fd9c91ce1266ae259a5fe1d58279e8ccffb692f8
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Aug 29 13:59:48 2018 +0100
Add /_partition/$partition/... endpoints
---
src/chttpd/src/chttpd_db.erl | 68 +++++++++++++++++++++---
src/chttpd/src/chttpd_handlers.erl | 10 +++-
src/chttpd/src/chttpd_httpd_handlers.erl | 6 ++-
src/chttpd/src/chttpd_view.erl | 17 +++++-
src/couch_mrview/src/couch_mrview.erl | 1 +
src/couch_mrview/src/couch_mrview_util.erl | 14 +++++
src/mango/src/mango_error.erl | 7 +++
src/mango/src/mango_httpd.erl | 85 ++++++++++++++++++++++++++----
src/mango/src/mango_httpd_handlers.erl | 6 ++-
9 files changed, 192 insertions(+), 22 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index eb5b63e..53889ed 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -18,7 +18,8 @@
db_req/2, couch_doc_open/4,handle_changes_req/2,
update_doc_result_to_json/1, update_doc_result_to_json/2,
handle_design_info_req/3, handle_view_cleanup_req/2,
- update_doc/4, http_code_from_status/1]).
+ update_doc/4, http_code_from_status/1,
+ handle_partition_req/2]).
-import(chttpd,
[send_json/2,send_json/3,send_json/4,send_method_not_allowed/2,
@@ -250,21 +251,72 @@ handle_view_cleanup_req(Req, Db) ->
ok = fabric:cleanup_index_files_all_nodes(Db),
send_json(Req, 202, {[{ok, true}]}).
+
+handle_partition_req(#httpd{
+ path_parts=[DbName, <<"_partition">>, Partition, _Design, Name, <<"_",_/binary>> = Action | _Rest]
+ }=Req, Db) ->
+
+ validate_partition_req(Req, Partition, DbName),
+ DDoc = get_design_doc(DbName, Name),
+ Partitioned = couch_mrview:get_partitioned_opt(DDoc#doc.body, true),
+
+ case Partitioned of
+ true ->
+ Handler = chttpd_handlers:partition_design_handler(Action, fun bad_action_partition_design_req/4),
+ Handler(Req, Db, DDoc, Partition);
+ false ->
+ throw({bad_request, <<"partition query is not supported in this design doc.">>})
+ end;
+
+handle_partition_req(#httpd{
+ path_parts=[DbName, <<"_partition">>, Partition, Action | _Rest]
+ }=Req, Db) ->
+ validate_partition_req(Req, Partition, DbName),
+ Handler = chttpd_handlers:partition_handler(Action, fun bad_action_partition_req/3),
+ Handler(Req, Db, Partition);
+
+handle_partition_req(_Req, _Db) ->
+ throw({bad_request, <<"missing partition key">>}).
+
+
+bad_action_partition_design_req(Req, _Db, _DDoc, _PartitionKey) ->
+ chttpd:send_error(Req, 404, <<"partition_error">>, <<"Invalid path.">>).
+
+
+bad_action_partition_req(Req, _Db, _PartitionKey) ->
+ chttpd:send_error(Req, 404, <<"partition_error">>, <<"Invalid path.">>).
+
+
+validate_partition_req(_Req, Partition, DbName) ->
+ couch_doc:validate_docid(Partition, DbName),
+
+ case mem3:is_partitioned(DbName) of
+ false -> throw({bad_request, <<"Database is not partitioned">>});
+ true -> ok
+ end.
+
+
handle_design_req(#httpd{
path_parts=[_DbName, _Design, Name, <<"_",_/binary>> = Action | _Rest]
}=Req, Db) ->
DbName = mem3:dbname(couch_db:name(Db)),
- case ddoc_cache:open(DbName, <<"_design/", Name/binary>>) of
- {ok, DDoc} ->
- Handler = chttpd_handlers:design_handler(Action, fun bad_action_req/3),
- Handler(Req, Db, DDoc);
- Error ->
- throw(Error)
- end;
+ DDoc = get_design_doc(DbName, Name),
+ Handler = chttpd_handlers:design_handler(Action, fun bad_action_req/3),
+ Handler(Req, Db, DDoc);
handle_design_req(Req, Db) ->
db_req(Req, Db).
+
+get_design_doc(DbName, Name) ->
+ case ddoc_cache:open(DbName, <<"_design/", Name/binary>>) of
+ {ok, DDoc} ->
+ DDoc;
+ Error ->
+ throw(Error)
+ end.
+
+
bad_action_req(#httpd{path_parts=[_, _, Name|FileNameParts]}=Req, Db, _DDoc) ->
db_attachment_req(Req, Db, <<"_design/",Name/binary>>, FileNameParts).
diff --git a/src/chttpd/src/chttpd_handlers.erl b/src/chttpd/src/chttpd_handlers.erl
index 9305632..f2098be 100644
--- a/src/chttpd/src/chttpd_handlers.erl
+++ b/src/chttpd/src/chttpd_handlers.erl
@@ -15,7 +15,9 @@
-export([
url_handler/2,
db_handler/2,
- design_handler/2
+ design_handler/2,
+ partition_handler/2,
+ partition_design_handler/2
]).
-define(SERVICE_ID, chttpd_handlers).
@@ -35,6 +37,12 @@ db_handler(HandlerKey, DefaultFun) ->
design_handler(HandlerKey, DefaultFun) ->
select(collect(design_handler, [HandlerKey]), DefaultFun).
+partition_handler(HandlerKey, DefaultFun) ->
+ select(collect(partition_handler, [HandlerKey]), DefaultFun).
+
+partition_design_handler(HandlerKey, DefaultFun) ->
+ select(collect(partition_design_handler, [HandlerKey]), DefaultFun).
+
%% ------------------------------------------------------------------
%% Internal Function Definitions
%% ------------------------------------------------------------------
diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl
index cb52e2c..2659d39 100644
--- a/src/chttpd/src/chttpd_httpd_handlers.erl
+++ b/src/chttpd/src/chttpd_httpd_handlers.erl
@@ -12,7 +12,7 @@
-module(chttpd_httpd_handlers).
--export([url_handler/1, db_handler/1, design_handler/1]).
+-export([url_handler/1, db_handler/1, design_handler/1, partition_design_handler/1]).
url_handler(<<>>) -> fun chttpd_misc:handle_welcome_req/1;
url_handler(<<"favicon.ico">>) -> fun chttpd_misc:handle_favicon_req/1;
@@ -32,6 +32,7 @@ url_handler(_) -> no_match.
db_handler(<<"_view_cleanup">>) -> fun chttpd_db:handle_view_cleanup_req/2;
db_handler(<<"_compact">>) -> fun chttpd_db:handle_compact_req/2;
db_handler(<<"_design">>) -> fun chttpd_db:handle_design_req/2;
+db_handler(<<"_partition">>) -> fun chttpd_db:handle_partition_req/2;
db_handler(<<"_temp_view">>) -> fun chttpd_view:handle_temp_view_req/2;
db_handler(<<"_changes">>) -> fun chttpd_db:handle_changes_req/2;
db_handler(_) -> no_match.
@@ -43,3 +44,6 @@ design_handler(<<"_update">>) -> fun chttpd_show:handle_doc_update_req/3;
design_handler(<<"_info">>) -> fun chttpd_db:handle_design_info_req/3;
design_handler(<<"_rewrite">>) -> fun chttpd_rewrite:handle_rewrite_req/3;
design_handler(_) -> no_match.
+
+partition_design_handler(<<"_view">>) -> fun chttpd_view:handle_partition_view_req/4;
+partition_design_handler(_) -> no_match.
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index 6b95706..627663c 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -14,7 +14,7 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
--export([handle_view_req/3, handle_temp_view_req/2]).
+-export([handle_view_req/3, handle_temp_view_req/2, handle_partition_view_req/4]).
multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
Args0 = couch_mrview_http:parse_params(Req, undefined),
@@ -42,6 +42,10 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
Args = couch_mrview_http:parse_params(Req, Keys),
+ design_doc_view_int(Req, Db, DDoc, ViewName, Args).
+
+
+design_doc_view_int(Req, Db, DDoc, ViewName, Args) ->
couch_mrview_util:validate_args(Args, [view]),
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
@@ -102,6 +106,17 @@ handle_temp_view_req(Req, _Db) ->
chttpd:send_error(Req, 410, gone, Msg).
+handle_partition_view_req(#httpd{method='GET',
+ path_parts=[_, _, _, _, _, _, ViewName]} = Req, Db, DDoc, Partition) ->
+ Keys = chttpd:qs_json_value(Req, "keys", undefined),
+ Args = couch_mrview_http:parse_params(Req, Keys),
+ Args1 = couch_mrview_util:set_extra(Args, partition, Partition),
+ Args2 = couch_mrview_util:set_extra(Args1, partitioned, true),
+ design_doc_view_int(Req, Db, DDoc, ViewName, Args2);
+
+handle_partition_view_req(Req, _Db, _DDoc, _Pk) ->
+ chttpd:send_method_not_allowed(Req, "GET").
+
-ifdef(TEST).
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 7862afb..1b1a06b 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -24,6 +24,7 @@
-export([refresh/2]).
-export([compact/2, compact/3, cancel_compaction/2]).
-export([cleanup/1]).
+-export([get_partitioned_opt/2]).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 794b694..0fa5c1f 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -617,6 +617,20 @@ validate_args(Args, ValidateOptions) ->
ok
end,
+ case {Partitioned, Args#mrargs.conflicts} of
+ {true, true} ->
+ mrverror(<<"`conflicts=true` is not supported in this view.">>);
+ {_, _} ->
+ ok
+ end,
+
+ case {Partitioned, Args#mrargs.stable} of
+ {true, true} ->
+ mrverror(<<"`stable=true` is not supported in this view.">>);
+ {_, _} ->
+ ok
+ end,
+
Args1 = case {Style, Partitioned, Partition} of
{all_docs, true, undefined} ->
Args;
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
index b2bbb39..603fb5f 100644
--- a/src/mango/src/mango_error.erl
+++ b/src/mango/src/mango_error.erl
@@ -73,6 +73,13 @@ info(mango_fields, {invalid_field_json, BadField}) ->
fmt("Invalid JSON for field spec: ~w", [BadField])
};
+info(mango_httpd, partition_field_error) ->
+ {
+ 400,
+ <<"bad request">>,
+ <<"`partition` is not a valid parameter.">>
+ };
+
info(mango_httpd, error_saving_ddoc) ->
{
500,
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 2e87771..9a5c266 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -14,7 +14,8 @@
-export([
- handle_req/2
+ handle_req/2,
+ handle_partition_req/3
]).
@@ -38,13 +39,7 @@ handle_req(#httpd{} = Req, Db0) ->
handle_req_int(Req, Db)
catch
throw:{mango_error, Module, Reason} ->
- case mango_error:info(Module, Reason) of
- {500, ErrorStr, ReasonStr} ->
- Stack = erlang:get_stacktrace(),
- chttpd:send_error(Req, {ErrorStr, ReasonStr, Stack});
- {Code, ErrorStr, ReasonStr} ->
- chttpd:send_error(Req, Code, ErrorStr, ReasonStr)
- end
+ handle_req_error(Req, Module, Reason)
end.
@@ -58,6 +53,34 @@ handle_req_int(_, _) ->
throw({not_found, missing}).
+handle_partition_req(#httpd{} = Req, Db0, Partition) ->
+ try
+ Db = set_user_ctx(Req, Db0),
+ handle_partition_req_int(Req, Db, Partition)
+ catch
+ throw:{mango_error, Module, Reason} ->
+ handle_req_error(Req, Module, Reason)
+ end.
+
+
+handle_partition_req_int(#httpd{path_parts=[_, _, _, <<"_explain">> | _]} = Req, Db, Partition) ->
+ handle_partition_explain_req(Req, Db, Partition);
+handle_partition_req_int(#httpd{path_parts=[_, _, _,<<"_find">> | _]} = Req, Db, Partition) ->
+ handle_partition_find_req(Req, Db, Partition);
+handle_partition_req_int(_, _, _) ->
+ throw({not_found, missing}).
+
+
+handle_req_error(Req, Module, Reason) ->
+ case mango_error:info(Module, Reason) of
+ {500, ErrorStr, ReasonStr} ->
+ Stack = erlang:get_stacktrace(),
+ chttpd:send_error(Req, {ErrorStr, ReasonStr, Stack});
+ {Code, ErrorStr, ReasonStr} ->
+ chttpd:send_error(Req, Code, ErrorStr, ReasonStr)
+ end.
+
+
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
@@ -170,7 +193,9 @@ handle_index_req(#httpd{path_parts=[_, _, _DDocId0, _Type, _Name]}=Req, _Db) ->
handle_explain_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
+ {Body0} = chttpd:json_body_obj(Req),
+ check_for_partition_param(Body0),
+ {ok, Opts0} = mango_opts:validate_find({Body0}),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
Resp = mango_crud:explain(Db, Sel, Opts),
chttpd:send_json(Req, Resp);
@@ -179,9 +204,23 @@ handle_explain_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
+handle_partition_explain_req(#httpd{method='POST'}=Req, Db, Partition) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Body} = add_partition_to_query(Req, Partition),
+ {ok, Opts0} = mango_opts:validate_find(Body),
+ {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
+ Resp = mango_crud:explain(Db, Sel, Opts),
+ chttpd:send_json(Req, Resp);
+
+handle_partition_explain_req(Req, _Db, _Partition) ->
+ chttpd:send_method_not_allowed(Req, "POST").
+
+
handle_find_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
+ {Body0} = chttpd:json_body_obj(Req),
+ check_for_partition_param(Body0),
+ {ok, Opts0} = mango_opts:validate_find({Body0}),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
{ok, Resp0} = start_find_resp(Req),
{ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
@@ -191,6 +230,32 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
+handle_partition_find_req(#httpd{method='POST'}=Req, Db, Partition) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Body} = add_partition_to_query(Req, Partition),
+ {ok, Opts0} = mango_opts:validate_find(Body),
+ {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
+ {ok, Resp0} = start_find_resp(Req),
+ {ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
+ end_find_resp(AccOut);
+
+handle_partition_find_req(Req, _Db, _Partition) ->
+ chttpd:send_method_not_allowed(Req, "POST").
+
+check_for_partition_param(Body) ->
+ case lists:keyfind(<<"partition">>, 1, Body) of
+ false -> ok;
+ _ -> ?MANGO_ERROR(partition_field_error)
+ end.
+
+
+add_partition_to_query(Req, Partition) ->
+ {Body0} = chttpd:json_body_obj(Req),
+ check_for_partition_param(Body0),
+ Body1 = [{<<"partition">>, Partition} | Body0],
+ {ok, {Body1}}.
+
+
set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
{ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
NewDb.
diff --git a/src/mango/src/mango_httpd_handlers.erl b/src/mango/src/mango_httpd_handlers.erl
index 80e5e27..8589b7e 100644
--- a/src/mango/src/mango_httpd_handlers.erl
+++ b/src/mango/src/mango_httpd_handlers.erl
@@ -12,7 +12,7 @@
-module(mango_httpd_handlers).
--export([url_handler/1, db_handler/1, design_handler/1]).
+-export([url_handler/1, db_handler/1, design_handler/1, partition_handler/1]).
url_handler(_) -> no_match.
@@ -22,3 +22,7 @@ db_handler(<<"_find">>) -> fun mango_httpd:handle_req/2;
db_handler(_) -> no_match.
design_handler(_) -> no_match.
+
+partition_handler(<<"_find">>) -> fun mango_httpd:handle_partition_req/3;
+partition_handler(<<"_explain">>) -> fun mango_httpd:handle_partition_req/3;
+partition_handler(_) -> no_match.
[couchdb] 03/13: expose partitioned flag in GET /dbname info
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6586fe84a02a604a1a002b7843973e7d1dc9d5db
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Aug 6 14:45:54 2018 +0100
expose partitioned flag in GET /dbname info
---
src/couch/src/couch_db.erl | 5 +++++
src/fabric/src/fabric_db_info.erl | 13 +++++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index 9bb68f8..a5c71d7 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -445,6 +445,10 @@ get_db_info(Db) ->
undefined -> null;
Else1 -> Else1
end,
+ Props = case get_props(Db) of
+ undefined -> null;
+ Else2 -> Else2
+ end,
InfoList = [
{db_name, Name},
{engine, couch_db_engine:get_engine(Db)},
@@ -466,6 +470,7 @@ get_db_info(Db) ->
{disk_format_version, DiskVersion},
{committed_update_seq, CommittedUpdateSeq},
{compacted_seq, CompactedSeq},
+ {props, {Props}},
{uuid, Uuid}
],
{ok, InfoList}.
diff --git a/src/fabric/src/fabric_db_info.erl b/src/fabric/src/fabric_db_info.erl
index 98e8e52..86947fd 100644
--- a/src/fabric/src/fabric_db_info.erl
+++ b/src/fabric/src/fabric_db_info.erl
@@ -107,6 +107,8 @@ merge_results(Info) ->
[{disk_format_version, lists:max(X)} | Acc];
(cluster, [X], Acc) ->
[{cluster, {X}} | Acc];
+ (props, X, Acc) ->
+ [{props, {merge_object(X)}} | Acc];
(_, _, Acc) ->
Acc
end, [{instance_start_time, <<"0">>}], Dict).
@@ -127,10 +129,17 @@ merge_object(Objects) ->
lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, D, Props)
end, orddict:new(), Objects),
orddict:fold(fun
- (Key, X, Acc) ->
- [{Key, lists:sum(X)} | Acc]
+ (Key, [X | _] = Xs, Acc) when is_integer(X) ->
+ [{Key, lists:sum(Xs)} | Acc];
+ (Key, [X | _] = Xs, Acc) when is_boolean(X) ->
+ [{Key, lists:all(fun all_true/1, Xs)} | Acc];
+ (_Key, _X, Acc) ->
+ Acc
end, [], Dict).
+all_true(true) -> true;
+all_true(_) -> false.
+
get_cluster_info(Shards) ->
Dict = lists:foldl(fun(#shard{range = R}, Acc) ->
dict:update_counter(R, 1, Acc)
[couchdb] 05/13: Select only the shard containing the partition,
if specified
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 418ceab96ef80582a7593a341e389f5318fae237
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Aug 6 19:16:09 2018 +0100
Select only the shard containing the partition, if specified
---
src/fabric/src/fabric_util.erl | 1 -
src/fabric/src/fabric_view.erl | 32 ++++++++++++++++++++++++++++----
2 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index dd4b80d..4d2f2c7 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -64,7 +64,6 @@ stream_start(Workers0, Keypos, StartFun, Replacements) ->
Timeout = request_timeout(),
case rexi_utils:recv(Workers0, Keypos, Fun, Acc, Timeout, infinity) of
{ok, #stream_acc{workers=Workers}} ->
- true = fabric_view:is_progress_possible(Workers),
AckedWorkers = fabric_dict:fold(fun(Worker, From, WorkerAcc) ->
rexi:stream_start(From),
[Worker | WorkerAcc]
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 69f4290..b4b8a8c 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -309,10 +309,34 @@ index_of(X, [X|_Rest], I) ->
index_of(X, [_|Rest], I) ->
index_of(X, Rest, I+1).
-get_shards(DbName, #mrargs{stable=true}) ->
- mem3:ushards(DbName);
-get_shards(DbName, #mrargs{stable=false}) ->
- mem3:shards(DbName).
+
+get_shards(DbName, #mrargs{} = Args) ->
+ Partitioned = couch_mrview_util:get_extra(Args, partitioned),
+ Partition = partition_docid(Args),
+ case {Args#mrargs.stable, Partitioned, Partition} of
+ {true, false, _} ->
+ mem3:ushards(DbName);
+ {true, true, undefined} ->
+ mem3:ushards(DbName);
+ {true, true, Partition} ->
+ mem3:ushards(DbName, Partition);
+ {false, false, _} ->
+ mem3:shards(DbName);
+ {false, true, undefined} ->
+ mem3:shards(DbName);
+ {false, true, Partition} ->
+ mem3:shards(DbName, Partition)
+ end.
+
+% create a fake docid within the specified partition.
+partition_docid(Args) ->
+ case couch_mrview_util:get_extra(Args, partition) of
+ undefined ->
+ undefined;
+ Partition when is_binary(Partition) ->
+ <<Partition/binary, ":foo">>
+ end.
+
maybe_update_others(DbName, DDoc, ShardsInvolved, ViewName,
#mrargs{update=lazy} = Args) ->
[couchdb] 02/13: Enforce partition:id format in doc ids
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e55bd2bbccde2c589e0d1d7acf289dd4fb649023
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Aug 2 14:41:03 2018 +0100
Enforce partition:id format in doc ids
---
src/chttpd/src/chttpd_db.erl | 9 +++++++--
src/couch/src/couch_doc.erl | 33 +++++++++++++++++++++++++++++----
src/couch/test/fixtures/test.couch | Bin 16482 -> 0 bytes
3 files changed, 36 insertions(+), 6 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 0905559..63d2907 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -758,7 +758,7 @@ db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) ->
db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
couch_httpd:validate_referer(Req),
- couch_doc:validate_docid(DocId, couch_db:name(Db)),
+ validate_docid(DocId, couch_db:name(Db)),
chttpd:validate_ctype(Req, "multipart/form-data"),
W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
@@ -1291,7 +1291,7 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
% check for the existence of the doc to handle the 404 case.
couch_doc_open(Db, DocId, nil, [])
end,
- couch_doc:validate_docid(DocId, couch_db:name(Db)),
+ validate_docid(DocId, couch_db:name(Db)),
#doc{id=DocId};
Rev ->
case fabric:open_revs(Db, DocId, [Rev], [{user_ctx,Ctx}]) of
@@ -1766,6 +1766,11 @@ bulk_get_json_error(DocId, Rev, Error, Reason) ->
{<<"error">>, Error},
{<<"reason">>, Reason}]}}]}).
+validate_docid(DocId, DbName) ->
+ Partitioned = mem3:is_partitioned(DbName),
+ Options = [{partitioned, Partitioned}],
+ couch_doc:validate_docid(DocId, DbName, Options).
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/couch/src/couch_doc.erl b/src/couch/src/couch_doc.erl
index f960ec5..f821696 100644
--- a/src/couch/src/couch_doc.erl
+++ b/src/couch/src/couch_doc.erl
@@ -16,7 +16,7 @@
-export([from_json_obj/1, from_json_obj_validate/1]).
-export([from_json_obj/2, from_json_obj_validate/2]).
-export([to_json_obj/2, has_stubs/1, merge_stubs/2]).
--export([validate_docid/1, validate_docid/2, get_validate_doc_fun/1]).
+-export([validate_docid/1, validate_docid/2, validate_docid/3, get_validate_doc_fun/1]).
-export([doc_from_multi_part_stream/2, doc_from_multi_part_stream/3]).
-export([doc_from_multi_part_stream/4]).
-export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
@@ -133,6 +133,13 @@ from_json_obj_validate(EJson) ->
from_json_obj_validate(EJson, DbName) ->
MaxSize = config:get_integer("couchdb", "max_document_size", 4294967296),
Doc = from_json_obj(EJson, DbName),
+ case is_binary(DbName) andalso mem3:is_partitioned(DbName) of
+ true ->
+ Options = [{partitioned, true}],
+ couch_doc:validate_docid(Doc#doc.id, DbName, Options);
+ false ->
+ ok
+ end,
case couch_ejson_size:encoded_size(Doc#doc.body) =< MaxSize of
true ->
validate_attachment_sizes(Doc#doc.atts),
@@ -199,12 +206,30 @@ parse_revs(_) ->
validate_docid(DocId, DbName) ->
- case DbName =:= ?l2b(config:get("mem3", "shards_db", "_dbs")) andalso
- lists:member(DocId, ?SYSTEM_DATABASES) of
+ validate_docid(DocId, DbName, []).
+
+validate_docid(DocId, DbName, Options) ->
+ SystemId = DbName =:= ?l2b(config:get("mem3", "shards_db", "_dbs")) andalso
+ lists:member(DocId, ?SYSTEM_DATABASES),
+ case SystemId of
true ->
ok;
false ->
- validate_docid(DocId)
+ Partitioned = couch_util:get_value(partitioned, Options, false),
+ case Partitioned of
+ true ->
+ case binary:split(DocId, <<":">>) of
+ [<<"_design/", _/binary>> | _Rest] ->
+ validate_docid(DocId);
+ [Partition, Rest] ->
+ ok = validate_docid(Partition),
+ validate_docid(Rest);
+ _ ->
+ throw({illegal_docid, <<"doc id must be of form partition:id">>})
+ end;
+ false ->
+ validate_docid(DocId)
+ end
end.
validate_docid(<<"">>) ->
diff --git a/src/couch/test/fixtures/test.couch b/src/couch/test/fixtures/test.couch
deleted file mode 100644
index 32c79af..0000000
Binary files a/src/couch/test/fixtures/test.couch and /dev/null differ
[couchdb] 09/13: Implement _all_docs and _find support
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b6fd9464e1dbc22d08bd68e3862226187f1695ea
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Aug 9 13:43:17 2018 +0100
Implement _all_docs and _find support
Co-authored-by: Garren Smith <ga...@gmail.com>
Co-authored-by: Robert Newson <rn...@apache.org>
---
src/chttpd/src/chttpd_db.erl | 7 +--
src/couch_mrview/src/couch_mrview.erl | 7 +--
src/couch_mrview/src/couch_mrview_updater.erl | 4 +-
src/couch_mrview/src/couch_mrview_util.erl | 75 ++++++++++++++++++++++-----
src/fabric/src/fabric_view.erl | 2 +-
src/fabric/src/fabric_view_all_docs.erl | 2 +-
src/mango/src/mango_cursor_view.erl | 29 +++++++++--
src/mango/src/mango_opts.erl | 6 +++
src/mem3/src/mem3.erl | 5 +-
9 files changed, 111 insertions(+), 26 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ddcad97..eb5b63e 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -684,12 +684,13 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
all_docs_view(Req, Db, Keys, OP) ->
Args0 = couch_mrview_http:parse_params(Req, Keys),
Args1 = Args0#mrargs{view_type=map},
- Args2 = couch_mrview_util:validate_args(Args1),
- Args3 = set_namespace(OP, Args2),
+ Args2 = couch_mrview_util:set_extra(Args1, style, all_docs),
+ Args3 = couch_mrview_util:validate_args(Args2),
+ Args4 = set_namespace(OP, Args3),
Options = [{user_ctx, Req#httpd.user_ctx}],
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db=Db, req=Req, threshold=Max},
- {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args3),
+ {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args4),
{ok, Resp#vacc.resp}.
db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) ->
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 09945f5..f5963e7 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -228,12 +228,13 @@ query_all_docs(Db, Args0, Callback, Acc) ->
couch_index_util:hexsig(couch_hash:md5_hash(term_to_binary(Info)))
end),
Args1 = Args0#mrargs{view_type=map},
- Args2 = couch_mrview_util:validate_args(Args1),
- {ok, Acc1} = case Args2#mrargs.preflight_fun of
+ Args2 = couch_mrview_util:set_extra(Args1, style, all_docs),
+ Args3 = couch_mrview_util:validate_args(Args2),
+ {ok, Acc1} = case Args3#mrargs.preflight_fun of
PFFun when is_function(PFFun, 2) -> PFFun(Sig, Acc);
_ -> {ok, Acc}
end,
- all_docs_fold(Db, Args2, Callback, Acc1).
+ all_docs_fold(Db, Args3, Callback, Acc1).
query_view(Db, DDoc, VName) ->
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 2b69eee..0c1a17c 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -315,7 +315,9 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
design_opts=DesignOpts
} = State,
- Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, false),
+ DbPartitioned = mem3:is_partitioned(State#mrst.db_name),
+ Partitioned = couch_util:get_value(<<"partitioned">>, DesignOpts, DbPartitioned),
+
Revs = dict:from_list(dict:fetch_keys(Log0)),
Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 574aac7..02e695d 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -39,8 +39,11 @@
-define(GET_VIEW_RETRY_COUNT, 1).
-define(GET_VIEW_RETRY_DELAY, 50).
-define(LOWEST_KEY, null).
--define(HIGHEST_KEY, {[{<<239, 191, 176>>, null}]}). % is {"\ufff0": null}
-
+-define(HIGHEST_KEY, {<<255, 255, 255, 255>>}).
+-define(PARTITION_START(P), <<P/binary, $:>>).
+-define(PARTITION_END(P), <<P/binary, $;>>).
+-define(LOWEST(A, B), (if A < B -> A; true -> B end)).
+-define(HIGHEST(A, B), (if A > B -> A; true -> B end)).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -583,20 +586,38 @@ validate_args(Args) ->
_ -> mrverror(<<"Invalid value for `sorted`.">>)
end,
- case {get_extra(Args, partitioned, false), get_extra(Args, partition)} of
- {true, undefined} ->
+ Style = get_extra(Args, style, normal),
+ Partitioned = get_extra(Args, partitioned, false),
+ Partition = get_extra(Args, partition),
+
+ case {Style, Partitioned, Partition} of
+ {all_docs, true, _} ->
+ ok; % _all_docs can be called with or without partition parameter.
+ {all_docs, false, undefined} ->
+ ok;
+ {all_docs, false, _Partition} ->
+ mrverror(<<"`partition` parameter is not supported in this db.">>);
+ {normal, true, undefined} ->
mrverror(<<"`partition` parameter is mandatory for queries to this view.">>);
- {true, _Partition} ->
+ {normal, true, _Partition} ->
ok;
- {false, undefined} ->
+ {normal, false, undefined} ->
ok;
- {false, _Partition} ->
+ {normal, false, _Partition} ->
mrverror(<<"`partition` parameter is not supported in this view.">>)
end,
- Args1 = case get_extra(Args, partitioned, false) of
- true -> apply_partition(Args);
- false -> Args
+ Args1 = case {Style, Partitioned, Partition} of
+ {all_docs, true, undefined} ->
+ Args;
+ {all_docs, true, Partition} ->
+ apply_partition(Args, all_docs);
+ {all_docs, false, _} ->
+ Args;
+ {normal, true, _} ->
+ apply_partition(Args, normal);
+ {normal, false, _} ->
+ Args
end,
Args1#mrargs{
@@ -617,9 +638,12 @@ determine_group_level(#mrargs{group=true, group_level=undefined}) ->
determine_group_level(#mrargs{group_level=GroupLevel}) ->
GroupLevel.
-apply_partition(#mrargs{} = Args0) ->
+apply_partition(#mrargs{} = Args0, Style) ->
Partition = get_extra(Args0, partition),
- apply_partition(Partition, Args0).
+ case Style of
+ normal -> apply_partition(Partition, Args0);
+ all_docs -> apply_all_docs_partition(Partition, Args0)
+ end;
apply_partition(_Partition, #mrargs{keys=[{p, _, _} | _]} = Args) ->
Args; % already applied
@@ -650,6 +674,33 @@ apply_partition(Partition, Args) ->
end_key = {p, Partition, EK0}
}.
+%% all_docs is special as it's not really a view and is already
+%% effectively partitioned as the partition is a prefix of all keys.
+apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=undefined, end_key=undefined} = Args) ->
+ Args#mrargs{start_key = ?PARTITION_START(Partition), end_key = ?PARTITION_END(Partition)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=undefined, end_key=undefined} = Args) ->
+ Args#mrargs{start_key = ?PARTITION_END(Partition), end_key = ?PARTITION_START(Partition)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=SK, end_key=undefined} = Args) ->
+ Args#mrargs{start_key = ?HIGHEST(?PARTITION_START(Partition), SK), end_key = ?PARTITION_END(Partition)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=SK, end_key=undefined} = Args) ->
+ Args#mrargs{start_key = ?LOWEST(?PARTITION_END(Partition), SK), end_key = ?PARTITION_START(Partition)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=undefined, end_key=EK} = Args) ->
+ Args#mrargs{start_key = ?PARTITION_START(Partition), end_key = ?LOWEST(?PARTITION_END(Partition), EK)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=undefined, end_key=EK} = Args) ->
+ Args#mrargs{start_key = ?PARTITION_END(Partition), end_key = ?HIGHEST(?PARTITION_START(Partition), EK)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=fwd, start_key=SK, end_key=EK} = Args) ->
+ Args#mrargs{start_key = ?HIGHEST(?PARTITION_START(Partition), SK), end_key = ?LOWEST(?PARTITION_END(Partition), EK)};
+
+apply_all_docs_partition(Partition, #mrargs{direction=rev, start_key=SK, end_key=EK} = Args) ->
+ Args#mrargs{start_key = ?LOWEST(?PARTITION_END(Partition), SK), end_key = ?HIGHEST(?PARTITION_START(Partition), EK)}.
+
+
check_range(#mrargs{start_key=undefined}, _Cmp) ->
ok;
check_range(#mrargs{end_key=undefined}, _Cmp) ->
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 844b44d..c0e2974 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -122,7 +122,7 @@ maybe_send_row(State) ->
user_acc = AccIn,
query_args = QueryArgs
} = State,
- Partitioned = couch_mrview_util:get_extra(QueryArgs, partitioned, false),
+ Partitioned = couch_mrview_util:get_extra(QueryArgs, partitioned),
case fabric_dict:any(0, Counters) of
true ->
{ok, State};
diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl
index ac16dac..d515ab8 100644
--- a/src/fabric/src/fabric_view_all_docs.erl
+++ b/src/fabric/src/fabric_view_all_docs.erl
@@ -118,7 +118,7 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) ->
#mrargs{limit = Limit, skip = Skip, update_seq = UpdateSeq} = QueryArgs,
State = #collector{
db_name = DbName,
- query_args = QueryArgs,
+ query_args = couch_mrview_util:set_extra(QueryArgs, style, all_docs),
callback = Callback,
counters = fabric_dict:init(Workers, 0),
skip = Skip,
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index dbea36e..708a73e 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -29,6 +29,7 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
-include("mango_cursor.hrl").
+-include("mango_idx.hrl").
-include("mango_idx_view.hrl").
create(Db, Indexes, Selector, Opts) ->
@@ -71,7 +72,8 @@ explain(Cursor) ->
{direction, Args#mrargs.direction},
{stable, Args#mrargs.stable},
{update, Args#mrargs.update},
- {conflicts, Args#mrargs.conflicts}
+ {conflicts, Args#mrargs.conflicts},
+ {extra, {Args#mrargs.extra}}
]}}].
@@ -93,14 +95,33 @@ maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx} = Cursor) ->
- #mrargs{
+base_args(#cursor{index = Idx, opts = Opts} = Cursor) ->
+ Args1 = #mrargs{
view_type = map,
reduce = false,
start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges),
end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
include_docs = true
- }.
+ },
+ Args2 = case mem3:is_partitioned(Idx#idx.dbname) of
+ true ->
+ Partition = couch_util:get_value(partition, Opts),
+ add_partition_opts(Args1, Partition);
+ false ->
+ Args1
+ end,
+ add_style(Idx, Args2).
+
+add_partition_opts(#mrargs{} = Args, <<>>) ->
+ Args;
+add_partition_opts(#mrargs{} = Args, Partition) ->
+ Args1 = couch_mrview_util:set_extra(Args, partitioned, true),
+ couch_mrview_util:set_extra(Args1, partition, Partition).
+
+add_style(#idx{def = all_docs}, Args) ->
+ couch_mrview_util:set_extra(Args, style, all_docs);
+add_style(_, Args) ->
+ Args.
execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFun, UserAcc) ->
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 7bae9c9..87d876a 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -81,6 +81,12 @@ validate_find({Props}) ->
{tag, selector},
{validator, fun validate_selector/1}
]},
+ {<<"partition">>, [
+ {tag, partition},
+ {optional, true},
+ {default, <<>>},
+ {validator, fun is_string/1}
+ ]},
{<<"use_index">>, [
{tag, use_index},
{optional, true},
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index bafb227..e3cd388 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -348,7 +348,10 @@ is_partitioned(DbName0) when is_binary(DbName0) ->
false
end;
-is_partitioned(Shards) when is_list(Shards) ->
+is_partitioned([#shard{} | _] = Shards) ->
+ lists:all(fun is_partitioned/1, Shards);
+
+is_partitioned([#ordered_shard{} | _] = Shards) ->
lists:all(fun is_partitioned/1, Shards);
is_partitioned(#shard{opts=Opts}) ->