You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2018/07/09 21:40:54 UTC
[couchdb] branch user-partitioned-dbs-wip updated (f1142df ->
28e0dfa)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch user-partitioned-dbs-wip
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard f1142df WIP support user-partitioned views
new 785213d WIP support user-partitioned views
new 28e0dfa WIP plumbing
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f1142df)
\
N -- N -- N refs/heads/user-partitioned-dbs-wip (28e0dfa)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch_mrview/include/couch_mrview.hrl | 3 ++-
src/couch_mrview/src/couch_mrview_util.erl | 30 ++++++++++++++++--------------
src/fabric/src/fabric.erl | 14 +++++++++-----
src/fabric/src/fabric_db_create.erl | 6 +++++-
src/mem3/src/mem3.erl | 10 +++++++++-
src/mem3/src/mem3_util.erl | 15 +++++++++++++--
6 files changed, 54 insertions(+), 24 deletions(-)
[couchdb] 02/02: WIP plumbing
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-wip
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 28e0dfa1f97c3b3dc9c2585d2a032868e49a11c0
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Jul 9 22:39:16 2018 +0100
WIP plumbing
---
src/couch_mrview/include/couch_mrview.hrl | 3 ++-
src/couch_mrview/src/couch_mrview_util.erl | 29 +++++++++++++++--------------
src/fabric/src/fabric.erl | 14 +++++++++-----
src/fabric/src/fabric_db_create.erl | 6 +++++-
src/mem3/src/mem3.erl | 10 +++++++++-
src/mem3/src/mem3_util.erl | 15 +++++++++++++--
6 files changed, 53 insertions(+), 24 deletions(-)
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index 67b3cd9..09faf51 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -32,7 +32,7 @@
doc_queue,
write_queue,
qserver=nil,
- partitioned=false
+ partitioned
}).
@@ -88,6 +88,7 @@
conflicts,
callback,
sorted = true,
+ partitioned,
partition_key,
extra = []
}).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 35554c4..aa67c88 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -24,7 +24,7 @@
-export([temp_view_to_ddoc/1]).
-export([calculate_external_size/1]).
-export([calculate_active_size/1]).
--export([validate_and_update_args/1, validate_and_update_args/2]).
+-export([validate_args/1, validate_and_update_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
-export([extract_view/4, extract_view_reduce/1]).
@@ -33,6 +33,7 @@
-export([changes_key_opts/2]).
-export([fold_changes/4]).
-export([to_key_seq/1]).
+-export([set_view_options/3]).
-export([partition_key/2, unpartition_key/1]).
-define(MOD, couch_mrview_index).
@@ -60,7 +61,7 @@ get_view(Db, DDoc, ViewName, Args0) ->
get_view_index_pid(Db, DDoc, ViewName, Args0) ->
ArgCheck = fun(InitState) ->
Args1 = set_view_type(Args0, ViewName, InitState#mrst.views),
- {ok, validate_and_update_args(Args1)}
+ {ok, validate_args(Args1)}
end,
couch_index_server:get_index(?MOD, Db, DDoc, ArgCheck).
@@ -449,7 +450,7 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
-validate_args(Args, Options) ->
+validate_args(Args) ->
GroupLevel = determine_group_level(Args),
Reduce = Args#mrargs.reduce,
case Reduce == undefined orelse is_boolean(Reduce) of
@@ -559,21 +560,20 @@ validate_args(Args, Options) ->
_ -> mrverror(<<"Invalid value for `sorted`.">>)
end,
- case {lists:member(partitioned, Options), Args#mrargs.partition_key} of
+ case {Args#mrargs.partitioned, Args#mrargs.partition_key} of
{true, undefined} ->
mrverror(<<"`partition_key` parameter is mandatory for queries to this database.">>);
{true, _PartitionKey} ->
ok;
- {false, undefined} ->
+ {undefined, undefined} ->
ok;
- {false, _PartitionKey} ->
+ {undefined, _PartitionKey} ->
mrverror(<<"`partition_key` parameter is not supported in this database.">>)
end,
+ Args.
- true.
-
-update_args(#mrargs{} = Args, _Options) ->
+update_args(#mrargs{} = Args) ->
GroupLevel = determine_group_level(Args),
SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of
@@ -621,11 +621,8 @@ update_args(#mrargs{} = Args, _Options) ->
validate_and_update_args(#mrargs{} = Args) ->
- validate_and_update_args(Args, []).
-
-validate_and_update_args(#mrargs{} = Args, Options) ->
- true = validate_args(Args, Options),
- update_args(Args, Options).
+ Args = validate_args(Args),
+ update_args(Args).
determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
@@ -1259,6 +1256,10 @@ kv_external_size(KVList, Reduction) ->
?term_size(Key) + ?term_size(Value) + Acc
end, ?term_size(Reduction), KVList).
+set_view_options(#mrargs{} = Args, partitioned, true) ->
+ Args#mrargs{partitioned=true};
+set_view_options(#mrargs{} = Args, partitioned, false) ->
+ Args#mrargs{partitioned=false}.
partition_key(Key, DocId) ->
[hd(binary:split(DocId, <<":">>)), Key].
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 1b761ab..59fbb2b 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -354,16 +354,20 @@ query_view(DbName, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) ->
end,
{ok, #mrst{views=Views, language=Lang}} =
couch_mrview_util:ddoc_to_mrst(Db, DDoc),
+
+ Partitioned = mem3:is_partitioned(hd(mem3:shards(Db))), % hideous
+
QueryArgs1 = couch_mrview_util:set_view_type(QueryArgs0, View, Views),
- QueryArgs2 = couch_mrview_util:validate_and_update_args(QueryArgs1),
- VInfo = couch_mrview_util:extract_view(Lang, QueryArgs2, View, Views),
- case is_reduce_view(QueryArgs2) of
+ QueryArgs2 = couch_mrview_util:set_view_options(QueryArgs1, partitioned, Partitioned),
+ QueryArgs3 = couch_mrview_util:validate_and_update_args(QueryArgs2),
+ VInfo = couch_mrview_util:extract_view(Lang, QueryArgs3, View, Views),
+ case is_reduce_view(QueryArgs3) of
true ->
fabric_view_reduce:go(
Db,
DDoc,
View,
- QueryArgs2,
+ QueryArgs3,
Callback,
Acc0,
VInfo
@@ -374,7 +378,7 @@ query_view(DbName, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) ->
Options,
DDoc,
View,
- QueryArgs2,
+ QueryArgs3,
Callback,
Acc0,
VInfo
diff --git a/src/fabric/src/fabric_db_create.erl b/src/fabric/src/fabric_db_create.erl
index 94ffd56..35e38cb 100644
--- a/src/fabric/src/fabric_db_create.erl
+++ b/src/fabric/src/fabric_db_create.erl
@@ -168,6 +168,10 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) ->
E when is_binary(E) -> [{<<"engine">>, E}];
_ -> []
end,
+ PartitionedProp = case lists:member(partitioned, Options) of
+ true -> [{<<"partitioned">>, true}];
+ false -> []
+ end,
#doc{
id = DbName,
body = {[
@@ -175,7 +179,7 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) ->
{<<"changelog">>, lists:sort(RawOut)},
{<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}},
{<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}}
- ] ++ EngineProp}
+ ] ++ EngineProp ++ PartitionedProp}
}.
db_exists(DbName) -> is_list(catch mem3:shards(DbName)).
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index ca083c4..f4bed76 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -23,7 +23,7 @@
-export([get_placement/1]).
%% For mem3 use only.
--export([name/1, node/1, range/1, engine/1]).
+-export([name/1, node/1, range/1, engine/1, is_partitioned/1]).
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch/include/couch_db.hrl").
@@ -331,6 +331,14 @@ engine(Opts) when is_list(Opts) ->
[]
end.
+is_partitioned(#shard{opts=Opts}) ->
+ is_partitioned(Opts);
+is_partitioned(#ordered_shard{opts=Opts}) ->
+ is_partitioned(Opts);
+is_partitioned(Opts) when is_list(Opts) ->
+ lists:member(partitioned, Opts).
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index fc6123d..5af8c95 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -177,7 +177,7 @@ build_shards_by_node(DbName, DocProps) ->
dbname = DbName,
node = to_atom(Node),
range = [Beg, End],
- opts = get_engine_opt(DocProps)
+ opts = get_opts(DocProps)
}, Suffix)
end, Ranges)
end, ByNode).
@@ -195,7 +195,7 @@ build_shards_by_range(DbName, DocProps) ->
node = to_atom(Node),
range = [Beg, End],
order = Order,
- opts = get_engine_opt(DocProps)
+ opts = get_opts(DocProps)
}, Suffix)
end, lists:zip(Nodes, lists:seq(1, length(Nodes))))
end, ByRange).
@@ -212,6 +212,9 @@ to_integer(N) when is_binary(N) ->
to_integer(N) when is_list(N) ->
list_to_integer(N).
+get_opts(DocProps) ->
+ get_engine_opt(DocProps) ++ get_partitioned_opt(DocProps).
+
get_engine_opt(DocProps) ->
case couch_util:get_value(<<"engine">>, DocProps) of
Engine when is_binary(Engine) ->
@@ -220,6 +223,14 @@ get_engine_opt(DocProps) ->
[]
end.
+get_partitioned_opt(DocProps) ->
+ case couch_util:get_value(<<"partitioned">>, DocProps) of
+ true ->
+ [partitioned];
+ _ ->
+ []
+ end.
+
n_val(undefined, NodeCount) ->
n_val(config:get("cluster", "n", "3"), NodeCount);
n_val(N, NodeCount) when is_list(N) ->
[couchdb] 01/02: WIP support user-partitioned views
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-wip
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 785213db2772e24485dc1a78d2ee17822a540124
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Jul 9 18:08:57 2018 +0100
WIP support user-partitioned views
---
src/couch_mrview/include/couch_mrview.hrl | 4 +-
src/couch_mrview/src/couch_mrview_http.erl | 2 +
src/couch_mrview/src/couch_mrview_updater.erl | 15 +++++-
src/couch_mrview/src/couch_mrview_util.erl | 66 ++++++++++++++++++++++++---
4 files changed, 78 insertions(+), 9 deletions(-)
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index a341e30..67b3cd9 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -31,7 +31,8 @@
doc_acc,
doc_queue,
write_queue,
- qserver=nil
+ qserver=nil,
+ partitioned=false
}).
@@ -87,6 +88,7 @@
conflicts,
callback,
sorted = true,
+ partition_key,
extra = []
}).
diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl
index 9dae1d8..5ff7285 100644
--- a/src/couch_mrview/src/couch_mrview_http.erl
+++ b/src/couch_mrview/src/couch_mrview_http.erl
@@ -582,6 +582,8 @@ parse_param(Key, Val, Args, IsDecoded) ->
Args#mrargs{callback=couch_util:to_binary(Val)};
"sorted" ->
Args#mrargs{sorted=parse_boolean(Val)};
+ "partition_key" ->
+ Args#mrargs{partition_key=couch_util:to_binary(Val)};
_ ->
BKey = couch_util:to_binary(Key),
BVal = couch_util:to_binary(Val),
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index 214f487..e1cc280 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -311,7 +311,8 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
#mrst{
id_btree=IdBtree,
log_btree=LogBtree,
- first_build=FirstBuild
+ first_build=FirstBuild,
+ partitioned=Partitioned
} = State,
Revs = dict:from_list(dict:fetch_keys(Log0)),
@@ -328,8 +329,15 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
_ -> update_log(LogBtree, Log, Revs, Seqs, FirstBuild)
end,
- UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
+ UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs0, SKVs}}) ->
#mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
+ KVs = case Partitioned of
+ true ->
+ [{{[partition(D), K], D}, V} || {{K, D}, V} <- KVs0];
+ false ->
+ KVs0
+ end,
+
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
@@ -484,3 +492,6 @@ maybe_notify(State, View, KVs, ToRem) ->
[Key || {Key, _DocId} <- ToRem]
end,
couch_index_plugin:index_update(State, View, Updated, Removed).
+
+partition(DocId) ->
+ hd(binary:split(DocId, <<":">>)).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 086bf9b..35554c4 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -24,7 +24,7 @@
-export([temp_view_to_ddoc/1]).
-export([calculate_external_size/1]).
-export([calculate_active_size/1]).
--export([validate_and_update_args/1]).
+-export([validate_and_update_args/1, validate_and_update_args/2]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
-export([extract_view/4, extract_view_reduce/1]).
@@ -33,6 +33,7 @@
-export([changes_key_opts/2]).
-export([fold_changes/4]).
-export([to_key_seq/1]).
+-export([partition_key/2, unpartition_key/1]).
-define(MOD, couch_mrview_index).
-define(GET_VIEW_RETRY_COUNT, 1).
@@ -281,6 +282,12 @@ init_state(Db, Fd, State, Header) ->
OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
Views2 = lists:zipwith(OpenViewFun, ViewStates, Views),
+ Partitioned = case couch_db_engine:get_prop(Db, partitioned) of
+ {ok, true} -> true;
+ {ok, false} -> false;
+ {error, no_value} -> false
+ end,
+
State#mrst{
fd=Fd,
fd_monitor=erlang:monitor(process, Fd),
@@ -288,7 +295,8 @@ init_state(Db, Fd, State, Header) ->
purge_seq=PurgeSeq,
id_btree=IdBtree,
log_btree=LogBtree,
- views=Views2
+ views=Views2,
+ partitioned=Partitioned
}.
open_view(_Db, Fd, Lang, ViewState, View) ->
@@ -441,7 +449,7 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
-validate_args(Args) ->
+validate_args(Args, Options) ->
GroupLevel = determine_group_level(Args),
Reduce = Args#mrargs.reduce,
case Reduce == undefined orelse is_boolean(Reduce) of
@@ -551,10 +559,21 @@ validate_args(Args) ->
_ -> mrverror(<<"Invalid value for `sorted`.">>)
end,
+ case {lists:member(partitioned, Options), Args#mrargs.partition_key} of
+ {true, undefined} ->
+ mrverror(<<"`partition_key` parameter is mandatory for queries to this database.">>);
+ {true, _PartitionKey} ->
+ ok;
+ {false, undefined} ->
+ ok;
+ {false, _PartitionKey} ->
+ mrverror(<<"`partition_key` parameter is not supported in this database.">>)
+ end,
+
true.
-update_args(#mrargs{} = Args) ->
+update_args(#mrargs{} = Args, _Options) ->
GroupLevel = determine_group_level(Args),
SKDocId = case {Args#mrargs.direction, Args#mrargs.start_key_docid} of
@@ -569,7 +588,32 @@ update_args(#mrargs{} = Args) ->
{_, EKDocId1} -> EKDocId1
end,
+ LowestKey = null,
+ HighestKey = {[{<<239, 191, 176>>, null}]}, % \ufff0
+
+ {StartKey, EndKey} = case Args of
+ #mrargs{partition_key=undefined} ->
+ {Args#mrargs.start_key, Args#mrargs.end_key};
+
+ #mrargs{partition_key=PKey0} when not is_binary(PKey0) ->
+ mrverror(<<"`partition_key` must be a string.">>);
+
+ #mrargs{partition_key=PKey0, start_key=undefined, end_key=undefined} ->
+ {[PKey0, LowestKey], [PKey0, HighestKey]};
+
+ #mrargs{partition_key=PKey0, start_key=SK0, end_key=undefined} ->
+ {[PKey0, SK0], [PKey0, HighestKey]};
+
+ #mrargs{partition_key=PKey0, start_key=undefined, end_key=EK0} ->
+ {[PKey0, LowestKey], [PKey0, EK0]};
+
+ #mrargs{partition_key=PKey0, start_key=SK0, end_key=EK0} ->
+ {[PKey0, SK0], [PKey0, EK0]}
+ end,
+
Args#mrargs{
+ start_key=StartKey,
+ end_key=EndKey,
start_key_docid=SKDocId,
end_key_docid=EKDocId,
group_level=GroupLevel
@@ -577,8 +621,11 @@ update_args(#mrargs{} = Args) ->
validate_and_update_args(#mrargs{} = Args) ->
- true = validate_args(Args),
- update_args(Args).
+ validate_and_update_args(Args, []).
+
+validate_and_update_args(#mrargs{} = Args, Options) ->
+ true = validate_args(Args, Options),
+ update_args(Args, Options).
determine_group_level(#mrargs{group=undefined, group_level=undefined}) ->
@@ -1211,3 +1258,10 @@ kv_external_size(KVList, Reduction) ->
lists:foldl(fun([[Key, _], Value], Acc) ->
?term_size(Key) + ?term_size(Value) + Acc
end, ?term_size(Reduction), KVList).
+
+
+partition_key(Key, DocId) ->
+ [hd(binary:split(DocId, <<":">>)), Key].
+
+unpartition_key([_Partition, Key]) ->
+ Key.