You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ro...@apache.org on 2015/02/03 16:13:45 UTC
[38/50] [abbrv] couchdb-mango git commit: Move view specific logic to
mango_cursor_view
Move view specific logic to mango_cursor_view
A good portion of mango_cursor turned out to be specific to view based
indexes. This moves the view related logic to mango_cursor_view and
updates mango_cursor to be more generic in preparation for the text
based indexing.
BugzId: 33294
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/06bf7757
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/06bf7757
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/06bf7757
Branch: refs/heads/master
Commit: 06bf7757b9f98c7cf5d5c597713021f6cb4ae27b
Parents: 7a9de34
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Jan 9 14:21:12 2015 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Fri Jan 16 13:32:49 2015 -0600
----------------------------------------------------------------------
src/mango_cursor.erl | 160 +++++++++--------------------------------
src/mango_cursor_view.erl | 71 ++++++++++++++++++
src/mango_error.erl | 4 +-
src/mango_idx.erl | 36 ++++++++++
src/mango_idx_special.erl | 6 ++
src/mango_idx_view.erl | 9 +++
6 files changed, 156 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_cursor.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 8f780b7..5bfe383 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -29,34 +29,24 @@
create(Db, Selector0, Opts) ->
Selector = mango_selector:normalize(Selector0),
- IndexFields = mango_idx_view:indexable_fields(Selector),
- FieldRanges = mango_idx_view:field_ranges(Selector, IndexFields),
-
- if IndexFields /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, operator_unsupported})
- end,
ExistingIndexes = mango_idx:list(Db),
- UsableIndexes = find_usable_indexes(IndexFields, ExistingIndexes),
- SortIndexes = get_sort_indexes(ExistingIndexes, UsableIndexes, Opts),
+ if ExistingIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, no_indexes_defined})
+ end,
- Composited = composite_indexes(SortIndexes, FieldRanges),
- {Index, Ranges} = choose_best_index(Db, Composited),
+ SortIndexes = mango_idx:for_sort(ExistingIndexes, Opts),
+ if SortIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index})
+ end,
- Limit = couch_util:get_value(limit, Opts, 10000000000),
- Skip = couch_util:get_value(skip, Opts, 0),
- Fields = couch_util:get_value(fields, Opts, all_fields),
+ UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
+ UsableIndexes = lists:filter(UsableFilter, SortIndexes),
+ if UsableIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, selector_unsupported})
+ end,
- {ok, #cursor{
- db = Db,
- index = Index,
- ranges = Ranges,
- selector = Selector,
- opts = Opts,
- limit = Limit,
- skip = Skip,
- fields = Fields
- }}.
+ create_cursor(Db, UsableIndexes, Selector, Opts).
execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
@@ -64,109 +54,23 @@ execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
Mod:execute(Cursor, UserFun, UserAcc).
-% Find the intersection between the Possible and Existing
-% indexes.
-find_usable_indexes([], _) ->
- ?MANGO_ERROR({no_usable_index, query_unsupported});
-find_usable_indexes(Possible, []) ->
- ?MANGO_ERROR({no_usable_index, {fields, Possible}});
-find_usable_indexes(Possible, Existing) ->
- Usable = lists:foldl(fun(Idx, Acc) ->
- [Col0 | _] = mango_idx:columns(Idx),
- case lists:member(Col0, Possible) of
- true ->
- [Idx | Acc];
- false ->
- Acc
+create_cursor(Db, Indexes, Selector, Opts) ->
+ [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
+ CursorMod:create(Db, CursorModIndexes, Selector, Opts).
+
+
+group_indexes_by_type(Indexes) ->
+ IdxDict = lists:foldl(fun(I, D) ->
+ dict:append(mango_idx:cursor_mod(I), I, D)
+ end, dict:new(), Indexes),
+ CursorModules = [
+ mango_cursor_view
+ ],
+ lists:flatmap(fun(CMod) ->
+ case dict:find(CMod, IdxDict) of
+ {ok, CModIndexes} ->
+ [{CMod, CModIndexes}];
+ error ->
+ []
end
- end, [], Existing),
- if length(Usable) > 0 -> ok; true ->
- ?MANGO_ERROR({no_usable_index, {fields, Possible}})
- end,
- Usable.
-
-
-get_sort_indexes(ExistingIndexes, UsableIndexes, Opts) ->
- % If a sort was specified we have to find an index that
- % can satisfy the request.
- case lists:keyfind(sort, 1, Opts) of
- {sort, {[_ | _]} = Sort} ->
- limit_to_sort(ExistingIndexes, UsableIndexes, Sort);
- _ ->
- UsableIndexes
- end.
-
-
-limit_to_sort(ExistingIndexes, UsableIndexes, Sort) ->
- Fields = mango_sort:fields(Sort),
-
- % First make sure that we have an index that could
- % answer this sort. We split like this so that the
- % user error is more obvious.
- SortFilt = fun(Idx) ->
- Cols = mango_idx:columns(Idx),
- lists:prefix(Fields, Cols)
- end,
- SortIndexes = lists:filter(SortFilt, ExistingIndexes),
- if SortIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, {sort, Fields}})
- end,
-
- % And then check if one or more of our SortIndexes
- % is usable.
- UsableFilt = fun(Idx) -> lists:member(Idx, UsableIndexes) end,
- FinalIndexes = lists:filter(UsableFilt, SortIndexes),
- if FinalIndexes /= [] -> ok; true ->
- ?MANGO_ERROR({no_usable_index, sort_field})
- end,
-
- FinalIndexes.
-
-
-% Any of these indexes may be a composite index. For each
-% index find the most specific set of fields for each
-% index. Ie, if an index has columns a, b, c, d, then
-% check FieldRanges for a, b, c, and d and return
-% the longest prefix of columns found.
-composite_indexes(Indexes, FieldRanges) ->
- lists:foldl(fun(Idx, Acc) ->
- Cols = mango_idx:columns(Idx),
- Prefix = composite_prefix(Cols, FieldRanges),
- [{Idx, Prefix} | Acc]
- end, [], Indexes).
-
-
-composite_prefix([], _) ->
- [];
-composite_prefix([Col | Rest], Ranges) ->
- case lists:keyfind(Col, 1, Ranges) of
- {Col, Range} ->
- [Range | composite_prefix(Rest, Ranges)];
- false ->
- []
- end.
-
-
-% Low and behold our query planner. Or something.
-% So stupid, but we can fix this up later. First
-% pass: Sort the IndexRanges by (num_columns, idx_name)
-% and return the first element. Yes. Its going to
-% be that dumb for now.
-%
-% In the future we can look into doing a cached parallel
-% reduce view read on each index with the ranges to find
-% the one that has the fewest number of rows or something.
-choose_best_index(_DbName, IndexRanges) ->
- Cmp = fun({A1, A2}, {B1, B2}) ->
- case length(A2) - length(B2) of
- N when N < 0 -> true;
- N when N == 0 ->
- % This is a really bad sort and will end
- % up preferring indices based on the
- % (dbname, ddocid, view_name) triple
- A1 =< B1;
- _ ->
- false
- end
- end,
- hd(lists:sort(Cmp, IndexRanges)).
+ end, CursorModules).
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_cursor_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_view.erl b/src/mango_cursor_view.erl
index a9f66b0..798b713 100644
--- a/src/mango_cursor_view.erl
+++ b/src/mango_cursor_view.erl
@@ -13,6 +13,7 @@
-module(mango_cursor_view).
-export([
+ create/4,
execute/3
]).
@@ -25,6 +26,27 @@
-include("mango_cursor.hrl").
+create(Db, Indexes, Selector, Opts) ->
+ FieldRanges = mango_idx_view:field_ranges(Selector),
+ Composited = composite_indexes(Indexes, FieldRanges),
+ {Index, IndexRanges} = choose_best_index(Db, Composited),
+
+ Limit = couch_util:get_value(limit, Opts, 10000000000),
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = IndexRanges,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
Cursor = Cursor0#cursor{
user_fun = UserFun,
@@ -52,6 +74,55 @@ execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
{ok, LastCursor#cursor.user_acc}.
+% Any of these indexes may be a composite index. For each
+% index find the most specific set of fields for each
+% index. Ie, if an index has columns a, b, c, d, then
+% check FieldRanges for a, b, c, and d and return
+% the longest prefix of columns found.
+composite_indexes(Indexes, FieldRanges) ->
+ lists:foldl(fun(Idx, Acc) ->
+ Cols = mango_idx:columns(Idx),
+ Prefix = composite_prefix(Cols, FieldRanges),
+ [{Idx, Prefix} | Acc]
+ end, [], Indexes).
+
+
+composite_prefix([], _) ->
+ [];
+composite_prefix([Col | Rest], Ranges) ->
+ case lists:keyfind(Col, 1, Ranges) of
+ {Col, Range} ->
+ [Range | composite_prefix(Rest, Ranges)];
+ false ->
+ []
+ end.
+
+
+% Low and behold our query planner. Or something.
+% So stupid, but we can fix this up later. First
+% pass: Sort the IndexRanges by (num_columns, idx_name)
+% and return the first element. Yes. Its going to
+% be that dumb for now.
+%
+% In the future we can look into doing a cached parallel
+% reduce view read on each index with the ranges to find
+% the one that has the fewest number of rows or something.
+choose_best_index(_DbName, IndexRanges) ->
+ Cmp = fun({A1, A2}, {B1, B2}) ->
+ case length(A2) - length(B2) of
+ N when N < 0 -> true;
+ N when N == 0 ->
+ % This is a really bad sort and will end
+ % up preferring indices based on the
+ % (dbname, ddocid, view_name) triple
+ A1 =< B1;
+ _ ->
+ false
+ end
+ end,
+ hd(lists:sort(Cmp, IndexRanges)).
+
+
handle_message({total_and_offset, _, _} = _TO, Cursor) ->
%twig:log(err, "TOTAL AND OFFSET: ~p", [_TO]),
{ok, Cursor};
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index 973dd43..778df2d 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -24,11 +24,11 @@ info(mango_cursor, {no_usable_index, operator_unsupported}) ->
<<"no_usable_index">>,
<<"There is no operator in this selector can used with an index.">>
};
-info(mango_cursor, {no_usable_index, query_unsupported}) ->
+info(mango_cursor, {no_usable_index, selector_unsupported}) ->
{
400,
<<"no_usable_index">>,
- <<"Query unsupported because it would require multiple indices.">>
+ <<"There is no index available for this selector.">>
};
info(mango_cursor, {no_usable_index, sort_field}) ->
{
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index c0a07a8..902fb75 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -20,6 +20,7 @@
-export([
list/1,
recover/1,
+ for_sort/2,
new/2,
validate/1,
@@ -35,6 +36,7 @@
def/1,
opts/1,
columns/1,
+ is_usable/2,
start_key/2,
end_key/2,
cursor_mod/1,
@@ -67,6 +69,35 @@ recover(Db) ->
end, DDocs)}.
+for_sort(Indexes, Opts) ->
+ % If a sort was specified we have to find an index that
+ % can satisfy the request.
+ case lists:keyfind(sort, 1, Opts) of
+ {sort, {SProps}} when is_list(SProps) ->
+ for_sort_int(Indexes, {SProps});
+ _ ->
+ Indexes
+ end.
+
+
+for_sort_int(Indexes, Sort) ->
+ Fields = mango_sort:fields(Sort),
+ FilterFun = fun(Idx) ->
+ Cols = mango_idx:columns(Idx),
+ case {mango_idx:type(Idx), Cols} of
+ {_, all_fields} ->
+ true;
+ {<<"text">>, _} ->
+ sets:is_subset(sets:from_list(Fields), sets:from_list(Cols));
+ {<<"json">>, _} ->
+ lists:prefix(Fields, Cols);
+ {<<"special">>, _} ->
+ lists:prefix(Fields, Cols)
+ end
+ end,
+ lists:filter(FilterFun, Indexes).
+
+
new(Db, Opts) ->
Def = get_idx_def(Opts),
Type = get_idx_type(Opts),
@@ -169,6 +200,11 @@ columns(#idx{}=Idx) ->
Mod:columns(Idx).
+is_usable(#idx{}=Idx, Selector) ->
+ Mod = idx_mod(Idx),
+ Mod:is_usable(Idx, Selector).
+
+
start_key(#idx{}=Idx, Ranges) ->
Mod = idx_mod(Idx),
Mod:start_key(Ranges).
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx_special.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_special.erl b/src/mango_idx_special.erl
index 0234dff..a8f9400 100644
--- a/src/mango_idx_special.erl
+++ b/src/mango_idx_special.erl
@@ -20,6 +20,7 @@
from_ddoc/1,
to_json/1,
columns/1,
+ is_usable/2,
start_key/1,
end_key/1
]).
@@ -62,6 +63,11 @@ columns(#idx{def=all_docs}) ->
[<<"_id">>].
+is_usable(#idx{def=all_docs}, Selector) ->
+ Fields = mango_idx_view:indexable_fields(Selector),
+ lists:member(<<"_id">>, Fields).
+
+
start_key([{'$gt', Key, _, _}]) ->
case mango_json:special(Key) of
true ->
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index eaa4341..ce0b206 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -19,6 +19,7 @@
remove/2,
from_ddoc/1,
to_json/1,
+ is_usable/2,
columns/1,
start_key/1,
end_key/1,
@@ -106,6 +107,14 @@ columns(Idx) ->
[Key || {Key, _} <- Fields].
+is_usable(Idx, Selector) ->
+ % This index is usable if at least the first column is
+ % a member of the indexable fields of the selector.
+ Columns = columns(Idx),
+ Fields = indexable_fields(Selector),
+ lists:member(hd(Columns), Fields).
+
+
start_key([]) ->
[];
start_key([{'$gt', Key, _, _} | Rest]) ->