You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2018/09/11 16:10:42 UTC
[couchdb] branch user-partitioned-dbs-6 updated (3e26f3b -> cb96435)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 3e26f3b allow partitioned:true in _index post. WIP
new cb96435 allow partitioned:true in _index post. merge somewhere
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3e26f3b)
\
N -- N -- N refs/heads/user-partitioned-dbs-6 (cb96435)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[couchdb] 01/01: allow partitioned:true in _index post. merge
somewhere
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch user-partitioned-dbs-6
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cb96435dfd0f99d11e7badfc6e8ec92ae339887a
Author: Robert Newson <rn...@apache.org>
AuthorDate: Mon Sep 10 11:56:56 2018 +0100
allow partitioned:true in _index post. merge somewhere
---
src/mango/src/mango_cursor_view.erl | 21 +++++++-----------
src/mango/src/mango_httpd.erl | 4 ++--
src/mango/src/mango_idx.erl | 44 +++++++++++++++++++++++++++++++++----
src/mango/src/mango_idx.hrl | 1 +
src/mango/src/mango_idx_view.erl | 16 +++++++++++++-
src/mango/src/mango_opts.erl | 10 +++++++--
6 files changed, 74 insertions(+), 22 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 708a73e..3eedf99 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -103,20 +103,15 @@ base_args(#cursor{index = Idx, opts = Opts} = Cursor) ->
end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
include_docs = true
},
- Args2 = case mem3:is_partitioned(Idx#idx.dbname) of
- true ->
- Partition = couch_util:get_value(partition, Opts),
- add_partition_opts(Args1, Partition);
- false ->
- Args1
+ Partitioned = couch_util:get_value(partitioned, Idx#idx.design_opts),
+ Args2 = couch_mrview_util:set_extra(Args1, partitioned, Partitioned),
+ Args3 = case couch_util:get_value(partition, Opts) of
+ <<>> ->
+ Args2;
+ Partition ->
+ couch_mrview_util:set_extra(Args2, partition, Partition)
end,
- add_style(Idx, Args2).
-
-add_partition_opts(#mrargs{} = Args, <<>>) ->
- Args;
-add_partition_opts(#mrargs{} = Args, Partition) ->
- Args1 = couch_mrview_util:set_extra(Args, partitioned, true),
- couch_mrview_util:set_extra(Args1, partition, Partition).
+ add_style(Idx, Args3).
add_style(#idx{def = all_docs}, Args) ->
couch_mrview_util:set_extra(Args, style, all_docs);
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index d9589b3..77a73c1 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -104,9 +104,9 @@ handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
JsonIdxs = lists:sublist(JsonIdxs0, Skip+1, Limit),
chttpd:send_json(Req, {[{total_rows, TotalRows}, {indexes, JsonIdxs}]});
-handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
+handle_index_req(#httpd{method='POST', path_parts=[DbName, _]}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
+ {ok, Opts} = mango_opts:validate_idx_create(DbName, chttpd:json_body_obj(Req)),
{ok, Idx0} = mango_idx:new(Db, Opts),
{ok, Idx} = mango_idx:validate_new(Idx0, Db),
DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body],
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 8af92b9..e051218 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -58,8 +58,9 @@ list(Db) ->
get_usable_indexes(Db, Selector, Opts) ->
- ExistingIndexes = mango_idx:list(Db),
-
+ PQ = is_partitioned_query(Opts),
+ ExistingIndexes = filter_indexes_by_partitioned(
+ mango_idx:list(Db), PQ),
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(ExistingIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
@@ -68,13 +69,36 @@ get_usable_indexes(Db, Selector, Opts) ->
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
case lists:filter(UsableFilter, UsableIndexes0) of
- [] ->
+ [] ->
?MANGO_ERROR({no_usable_index, missing_sort_index});
- UsableIndexes ->
+ UsableIndexes ->
UsableIndexes
end.
+filter_indexes_by_partitioned(Indexes, PQ) ->
+ filter_indexes_by_partitioned(Indexes, PQ, []).
+
+
+filter_indexes_by_partitioned([], _PQ, Acc) ->
+ lists:reverse(Acc);
+filter_indexes_by_partitioned([Idx | Rest], PQ, Acc) ->
+ {partitioned, PI} = lists:keyfind(partitioned, 1, Idx#idx.design_opts),
+ case {Idx#idx.def, PI, PQ} of
+ {all_docs, _, _} ->
+ % all_docs works both ways.
+ filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]);
+ {_, Same, Same} ->
+ filter_indexes_by_partitioned(Rest, PQ, [Idx | Acc]);
+ {_, _, _} ->
+ filter_indexes_by_partitioned(Rest, PQ, Acc)
+ end.
+
+
+is_partitioned_query(Opts) ->
+ lists:keyfind(partition, 1, Opts) /= {partition, <<>>}.
+
+
recover(Db) ->
{ok, DDocs0} = mango_util:open_ddocs(Db),
Pred = fun({Props}) ->
@@ -101,6 +125,7 @@ get_sort_fields(Opts) ->
new(Db, Opts) ->
Def = get_idx_def(Opts),
+ DesignOpts = get_idx_design_opts(Db, Opts),
Type = get_idx_type(Opts),
IdxName = get_idx_name(Def, Opts),
DDoc = get_idx_ddoc(Def, Opts),
@@ -110,6 +135,7 @@ new(Db, Opts) ->
name = IdxName,
type = Type,
def = Def,
+ design_opts = DesignOpts,
opts = filter_opts(Opts)
}}.
@@ -182,11 +208,13 @@ from_ddoc(Db, {Props}) ->
special(Db) ->
+ Partitioned = mem3:is_partitioned(Db),
AllDocs = #idx{
dbname = db_to_name(Db),
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
+ design_opts = [{partitioned, Partitioned}],
opts = []
},
% Add one for _update_seq
@@ -285,6 +313,12 @@ get_idx_def(Opts) ->
end.
+get_idx_design_opts(Db, Opts) ->
+ DbPartitioned = mem3:is_partitioned(couch_db:name(Db)),
+ Partitioned = proplists:get_value(partitioned, Opts, DbPartitioned),
+ [{partitioned, Partitioned}].
+
+
get_idx_type(Opts) ->
case proplists:get_value(type, Opts) of
<<"json">> -> <<"json">>;
@@ -341,6 +375,8 @@ filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{w, _} | Rest]) ->
filter_opts(Rest);
+filter_opts([{partitioned, _} | Rest]) ->
+ filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..179911f 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,6 @@
name,
type,
def,
+ design_opts,
opts
}).
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 2d784b6..ef1ca59 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -54,7 +54,8 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
NewView = make_view(Idx),
Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
- {ok, DDoc#doc{body={Props1}}}.
+ Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Idx#idx.design_opts}}),
+ {ok, DDoc#doc{body={Props2}}}.
remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -78,6 +79,7 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
from_ddoc({Props}) ->
+ DesignOpts = validate_design_opts(Props),
case lists:keyfind(<<"views">>, 1, Props) of
{<<"views">>, {Views}} when is_list(Views) ->
lists:flatmap(fun({Name, {VProps}}) ->
@@ -89,6 +91,7 @@ from_ddoc({Props}) ->
type = <<"json">>,
name = Name,
def = Def,
+ design_opts = DesignOpts,
opts = Opts
},
[I]
@@ -104,6 +107,7 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
+ {design_opts, {Idx#idx.design_opts}},
{def, {def_to_json(Idx#idx.def)}}
]}.
@@ -248,6 +252,16 @@ validate_ddoc(VProps) ->
end.
+validate_design_opts(Props) ->
+ case lists:keyfind(<<"options">>, 1, Props) of
+ {<<"options">>, {[{<<"partitioned">>, P}]}}
+ when is_boolean(P) ->
+ [{partitioned, P}];
+ _ ->
+ []
+ end.
+
+
% This function returns a list of indexes that
% can be used to restrict this query. This works by
% searching the selector looking for field names that
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index f1b7b60..365e248 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -13,7 +13,7 @@
-module(mango_opts).
-export([
- validate_idx_create/1,
+ validate_idx_create/2,
validate_find/1
]).
@@ -43,11 +43,17 @@
-include("mango.hrl").
-validate_idx_create({Props}) ->
+validate_idx_create(DbName, {Props}) ->
Opts = [
{<<"index">>, [
{tag, def}
]},
+ {<<"partitioned">>, [
+ {tag, partitioned},
+ {optional, true},
+ {default, mem3:is_partitioned(DbName)},
+ {validator, fun is_boolean/1}
+ ]},
{<<"type">>, [
{tag, type},
{optional, true},