You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2018/10/30 21:25:35 UTC
[couchdb] 01/01: Support partitioned queries in Mango
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature/user-partitioned-databases-davisp
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f44978cb6d75496354d3690e33cb3dabfd2ae5a1
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Mon Oct 29 14:08:05 2018 -0500
Support partitioned queries in Mango
Co-authored-by: Garren Smith <ga...@gmail.com>
Co-authored-by: Robert Newson <rn...@apache.org>
---
src/mango/src/mango_cursor.erl | 1 +
src/mango/src/mango_cursor_view.erl | 6 +++
src/mango/src/mango_error.erl | 7 +++
src/mango/src/mango_httpd.erl | 21 +++++++-
src/mango/src/mango_idx.erl | 105 +++++++++++++++++++++++++++++++++---
src/mango/src/mango_idx.hrl | 1 +
src/mango/src/mango_opts.erl | 23 ++++++++
7 files changed, 157 insertions(+), 7 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 5d2ea71..c6f21dd 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -71,6 +71,7 @@ explain(#cursor{}=Cursor) ->
{[
{dbname, mango_idx:dbname(Idx)},
{index, mango_idx:to_json(Idx)},
+ {partitioned, mango_idx:partitioned(Idx)},
{selector, Selector},
{opts, {Opts}},
{limit, Limit},
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 174381e..fc3eb0d 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -71,6 +71,7 @@ explain(Cursor) ->
{include_docs, Args#mrargs.include_docs},
{view_type, Args#mrargs.view_type},
{reduce, Args#mrargs.reduce},
+ {partition, couch_mrview_util:get_extra(Args, partition)},
{start_key, maybe_replace_max_json(Args#mrargs.start_key)},
{end_key, maybe_replace_max_json(Args#mrargs.end_key)},
{direction, Args#mrargs.direction},
@@ -395,6 +396,11 @@ apply_opts([{update, false} | Rest], Args) ->
update = false
},
apply_opts(Rest, NewArgs);
+apply_opts([{partition, <<>>} | Rest], Args) ->
+ apply_opts(Rest, Args);
+apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
+ NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
+ apply_opts(Rest, NewArgs);
apply_opts([{_, _} | Rest], Args) ->
% Ignore unknown options
apply_opts(Rest, Args).
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
index b2bbb39..b44ff68 100644
--- a/src/mango/src/mango_error.erl
+++ b/src/mango/src/mango_error.erl
@@ -104,6 +104,13 @@ info(mango_idx, {invalid_index_type, BadType}) ->
<<"invalid_index">>,
fmt("Invalid type for index: ~s", [BadType])
};
+info(mango_idx, {partitoned_option_mismatch, BadDDoc}) ->
+ {
+ 400,
+ <<"invalid_partitioned_option">>,
+ fmt("Requested partitioned option does not match existing value on"
+ " design document ~s", [BadDDoc])
+ };
info(mango_idx, invalid_query_ddoc_language) ->
{
400,
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 2e87771..dbe7cad 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -181,7 +181,9 @@ handle_explain_req(Req, _Db) ->
handle_find_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
+ Body = maybe_set_partition(Req),
+ {ok, Opts0} = mango_opts:validate_find(Body),
+ couch_log:error("~n~nXKCD: ~p~n~n", [Opts0]),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
{ok, Resp0} = start_find_resp(Req),
{ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
@@ -224,6 +226,23 @@ get_idx_del_opts(Req) ->
end.
+maybe_set_partition(Req) ->
+ {Props} = chttpd:json_body_obj(Req),
+ case chttpd:qs_value(Req, "partition", undefined) of
+ undefined ->
+ {Props};
+ Partition ->
+ case couch_util:get_value(<<"partition">>, Props) of
+ undefined ->
+ {[{<<"partition">>, ?l2b(Partition)} | Props]};
+ Partition ->
+ {Props};
+ OtherPartition ->
+ ?MANGO_ERROR({bad_partition, OtherPartition})
+ end
+ end.
+
+
convert_to_design_id(DDocId) ->
case DDocId of
<<"_design/", _/binary>> -> DDocId;
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 8af92b9..d07706a 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -33,6 +33,7 @@
name/1,
type/1,
def/1,
+ partitioned/1,
opts/1,
columns/1,
is_usable/3,
@@ -59,8 +60,10 @@ list(Db) ->
get_usable_indexes(Db, Selector, Opts) ->
ExistingIndexes = mango_idx:list(Db),
-
- GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes),
+ MatchingPartitionIndexes = filter_partition_indexes(ExistingIndexes, Opts),
+ GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
+ MatchingPartitionIndexes
+ ),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
@@ -110,6 +113,7 @@ new(Db, Opts) ->
name = IdxName,
type = Type,
def = Def,
+ partitioned = get_idx_partitioned(Opts),
opts = filter_opts(Opts)
}}.
@@ -121,10 +125,11 @@ validate_new(Idx, Db) ->
add(DDoc, Idx) ->
Mod = idx_mod(Idx),
- {ok, NewDDoc} = Mod:add(DDoc, Idx),
+ {ok, NewDDoc1} = Mod:add(DDoc, Idx),
+ NewDDoc2 = set_ddoc_partitioned(NewDDoc1, Idx),
% Round trip through JSON for normalization
- Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
- {ok, NewDDoc#doc{body = Body}}.
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)),
+ {ok, NewDDoc2#doc{body = Body}}.
remove(DDoc, Idx) ->
@@ -176,7 +181,8 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc
+ ddoc = DDoc,
+ partitioned = set_idx_partitioned(Db, Props)
}
end, Idxs).
@@ -213,6 +219,10 @@ def(#idx{def=Def}) ->
Def.
+partitioned(#idx{partitioned=Partitioned}) ->
+ Partitioned.
+
+
opts(#idx{opts=Opts}) ->
Opts.
@@ -329,6 +339,87 @@ gen_name(Idx, Opts0) ->
mango_util:enc_hex(Sha).
+get_idx_partitioned(Opts) ->
+ case proplists:get_value(partitioned, Opts) of
+ B when is_boolean(B) ->
+ B;
+ default ->
+ undefined
+ end.
+
+
+set_ddoc_partitioned(DDoc, Idx) ->
+ % We have to verify that the new index being added
+ % to this design document either matches the current
+ % ddoc's design options *or* this is a new design doc
+ #doc{
+ id = DDocId,
+ revs = Revs,
+ body = {BodyProps}
+ } = DDoc,
+ OldDOpts = couch_util:get_value(<<"options">>, BodyProps),
+ OldOpt = case OldDOpts of
+ {OldDOptProps} when is_list(OldDOptProps) ->
+ couch_util:get_value(<<"partitioned">>, OldDOptProps);
+ _ ->
+ undefined
+ end,
+ % If new matches old we're done
+ if Idx#idx.partitioned == OldOpt -> DDoc; true ->
+ % If we're creating a ddoc then we can set the options
+ case Revs == {0, []} of
+ true when Idx#idx.partitioned /= undefined ->
+ set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned);
+ true when Idx#idx.partitioned == undefined ->
+ DDoc;
+ false ->
+ ?MANGO_ERROR({partitioned_option_mismatch, DDocId})
+ end
+ end.
+
+
+set_ddoc_partitioned_option(DDoc, Partitioned) ->
+ #doc{
+ body = {BodyProps}
+ } = DDoc,
+ NewProps = case couch_util:get_value(<<"options">>, BodyProps) of
+ {Existing} when is_list(Existing) ->
+ Opt = {<<"partitioned">>, Partitioned},
+ New = lists:keystore(<<"partitioned">>, 1, Existing, Opt),
+ lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New});
+ undefined ->
+ New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}},
+ lists:keystore(<<"options">>, 1, BodyProps, New)
+ end,
+ DDoc#doc{body = {NewProps}}.
+
+
+set_idx_partitioned(Db, DDocProps) ->
+ Default = fabric_util:is_partitioned(Db),
+ case couch_util:get_value(<<"options">>, DDocProps) of
+ {DesignOpts} ->
+ case couch_util:get_value(<<"partitioned">>, DesignOpts) of
+ P when is_boolean(P) ->
+ P;
+ undefined ->
+ Default
+ end;
+ undefined ->
+ Default
+ end.
+
+
+filter_partition_indexes(Indexes, Opts) ->
+ PFilt = case couch_util:get_value(partition, Opts) of
+ Partition when is_binary(Partition) ->
+ fun(#idx{partitioned = P}) -> P end;
+ undefined ->
+ fun(#idx{partitioned = P}) -> not P end
+ end,
+ Filt = fun(Idx) -> type(Idx) == <<"special">> orelse PFilt(Idx) end,
+ lists:filter(Filt, Indexes).
+
+
filter_opts([]) ->
[];
filter_opts([{user_ctx, _} | Rest]) ->
@@ -341,6 +432,8 @@ filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{w, _} | Rest]) ->
filter_opts(Rest);
+filter_opts([{partitioned, _} | Rest]) ->
+ filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..9725950 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,6 @@
name,
type,
def,
+ partitioned,
opts
}).
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 7bae9c9..62a8832 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -34,6 +34,7 @@
validate_sort/1,
validate_fields/1,
validate_bulk_delete/1,
+ validate_partitioned/1,
default_limit/0
]).
@@ -70,6 +71,12 @@ validate_idx_create({Props}) ->
{optional, true},
{default, 2},
{validator, fun is_pos_integer/1}
+ ]},
+ {<<"partitioned">>, [
+ {tag, partitioned},
+ {optional, true},
+ {default, default},
+ {validator, fun validate_partitioned/1}
]}
],
validate(Props, Opts).
@@ -117,6 +124,12 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
+ {<<"partition">>, [
+ {tag, partition},
+ {optional, true},
+ {default, <<>>},
+ {validator, fun is_string/1}
+ ]},
{<<"r">>, [
{tag, r},
{optional, true},
@@ -296,6 +309,16 @@ validate_fields(Value) ->
mango_fields:new(Value).
+validate_partitioned(true) ->
+ {ok, true};
+validate_partitioned(false) ->
+ {ok, false};
+validate_partitioned(default) ->
+ {ok, default};
+validate_partitioned(Else) ->
+ ?MANGO_ERROR({invalid_partitioned_value, Else}).
+
+
validate_opts([], Props, Acc) ->
{Props, lists:reverse(Acc)};
validate_opts([{Name, Desc} | Rest], Props, Acc) ->