You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2018/06/08 20:36:44 UTC
[couchdb] branch master updated (8a46473 -> 398ac18)
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 8a46473 Add hyper app to dependencies
new 0392c51 Finalize in couch_mrview, but make it optional
new 5fa3c43 Use finalize operation to simplify _stats
new 398ac18 Ignore trailing characters in a builtin reduce
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch/src/couch_query_servers.erl | 31 +++++++++++++++++--------------
src/couch_mrview/src/couch_mrview.erl | 28 +++++++++++++++++++++++-----
src/fabric/src/fabric_rpc.erl | 5 +++--
src/fabric/src/fabric_view.erl | 2 +-
4 files changed, 44 insertions(+), 22 deletions(-)
--
To stop receiving notification emails like this one, please contact
kocolosk@apache.org.
[couchdb] 01/03: Finalize in couch_mrview, but make it optional
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0392c51dd67210bc70fb9f3ce8b2f191d3ad63ca
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Wed Jun 6 22:08:40 2018 -0400
Finalize in couch_mrview, but make it optional
If we're in a cluster, finalization runs at the coordinator. Otherwise,
couch_mrview can run it directly to simplify things for consumers.
---
src/couch_mrview/src/couch_mrview.erl | 28 +++++++++++++++++++++++-----
src/fabric/src/fabric_rpc.erl | 5 +++--
2 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index b417aac..82bbd79 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -41,6 +41,7 @@
user_acc,
last_go=ok,
reduce_fun,
+ finalizer,
update_seq,
args
}).
@@ -579,7 +580,14 @@ map_fold(#doc{id = <<"_local/", _/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
last_go=Go
}}.
-red_fold(Db, {_Nth, _Lang, View}=RedView, Args, Callback, UAcc) ->
+red_fold(Db, {NthRed, _Lang, View}=RedView, Args, Callback, UAcc) ->
+ Finalizer = case couch_util:get_value(finalizer, Args#mrargs.extra) of
+ undefined ->
+ {_, FunSrc} = lists:nth(NthRed, View#mrview.reduce_funs),
+ FunSrc;
+ CustomFun->
+ CustomFun
+ end,
Acc = #mracc{
db=Db,
total_rows=null,
@@ -589,6 +597,7 @@ red_fold(Db, {_Nth, _Lang, View}=RedView, Args, Callback, UAcc) ->
callback=Callback,
user_acc=UAcc,
update_seq=View#mrview.update_seq,
+ finalizer=Finalizer,
args=Args
},
Grouping = {key_group_level, Args#mrargs.group_level},
@@ -620,41 +629,50 @@ red_fold(_Key, _Red, #mracc{limit=0} = Acc) ->
{stop, Acc};
red_fold(_Key, Red, #mracc{group_level=0} = Acc) ->
#mracc{
+ finalizer=Finalizer,
limit=Limit,
callback=Callback,
user_acc=UAcc0
} = Acc,
- Row = [{key, null}, {value, Red}],
+ Row = [{key, null}, {value, maybe_finalize(Red, Finalizer)}],
{Go, UAcc1} = Callback({row, Row}, UAcc0),
{Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}};
red_fold(Key, Red, #mracc{group_level=exact} = Acc) ->
#mracc{
+ finalizer=Finalizer,
limit=Limit,
callback=Callback,
user_acc=UAcc0
} = Acc,
- Row = [{key, Key}, {value, Red}],
+ Row = [{key, Key}, {value, maybe_finalize(Red, Finalizer)}],
{Go, UAcc1} = Callback({row, Row}, UAcc0),
{Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}};
red_fold(K, Red, #mracc{group_level=I} = Acc) when I > 0, is_list(K) ->
#mracc{
+ finalizer=Finalizer,
limit=Limit,
callback=Callback,
user_acc=UAcc0
} = Acc,
- Row = [{key, lists:sublist(K, I)}, {value, Red}],
+ Row = [{key, lists:sublist(K, I)}, {value, maybe_finalize(Red, Finalizer)}],
{Go, UAcc1} = Callback({row, Row}, UAcc0),
{Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}};
red_fold(K, Red, #mracc{group_level=I} = Acc) when I > 0 ->
#mracc{
+ finalizer=Finalizer,
limit=Limit,
callback=Callback,
user_acc=UAcc0
} = Acc,
- Row = [{key, K}, {value, Red}],
+ Row = [{key, K}, {value, maybe_finalize(Red, Finalizer)}],
{Go, UAcc1} = Callback({row, Row}, UAcc0),
{Go, Acc#mracc{user_acc=UAcc1, limit=Limit-1, last_go=Go}}.
+maybe_finalize(Red, null) ->
+ Red;
+maybe_finalize(Red, RedSrc) ->
+ {ok, Finalized} = couch_query_servers:finalize(RedSrc, Red),
+ Finalized.
finish_fold(#mracc{last_go=ok, update_seq=UpdateSeq}=Acc, ExtraMeta) ->
#mracc{callback=Callback, user_acc=UAcc, args=Args}=Acc,
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 4a69e7e..913aafe0 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -142,8 +142,9 @@ reduce_view(DbName, DDoc, ViewName, Args0, DbOptions) ->
couch_mrview:query_view(Db, DDoc, ViewName, Args, fun reduce_cb/2, VAcc0).
fix_skip_and_limit(Args) ->
- #mrargs{skip=Skip, limit=Limit}=Args,
- Args#mrargs{skip=0, limit=Skip+Limit}.
+ #mrargs{skip=Skip, limit=Limit, extra=Extra}=Args,
+ % the coordinator needs to finalize each row, so make sure the shards don't
+ Args#mrargs{skip=0, limit=Skip+Limit, extra=[{finalizer,null} | Extra]}.
create_db(DbName) ->
create_db(DbName, []).
--
To stop receiving notification emails like this one, please contact
kocolosk@apache.org.
[couchdb] 02/03: Use finalize operation to simplify _stats
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5fa3c43f2c7313b18a63c0100ab2b1843bd8ab94
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Wed Jun 6 18:10:12 2018 -0400
Use finalize operation to simplify _stats
Rather than packing the stats into an ejson object on each write
we use a more compact tuple format on disk and then turn it into
ejson at query time.
---
src/couch/src/couch_query_servers.erl | 31 +++++++++++++++++--------------
src/fabric/src/fabric_view.erl | 2 +-
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
index 02d90f1..fe04533 100644
--- a/src/couch/src/couch_query_servers.erl
+++ b/src/couch/src/couch_query_servers.erl
@@ -17,7 +17,7 @@
-export([reduce/3, rereduce/3,validate_doc_update/5]).
-export([filter_docs/5]).
-export([filter_view/3]).
--export([finalize/1]).
+-export([finalize/2]).
-export([rewrite/3]).
-export([with_ddoc_proc/2, proc_prompt/2, ddoc_prompt/3, ddoc_proc_prompt/3, json_doc/1]).
@@ -87,15 +87,16 @@ group_reductions_results(List) ->
[Heads | group_reductions_results(Tails)]
end.
-finalize(Reductions) ->
- {ok, lists:map(fun(Reduction) ->
- case hyper:is_hyper(Reduction) of
- true ->
- round(hyper:card(Reduction));
- false ->
- Reduction
- end
- end, Reductions)}.
+finalize(<<"_approx_count_distinct">>, Reduction) ->
+ true = hyper:is_hyper(Reduction),
+ {ok, round(hyper:card(Reduction))};
+finalize(<<"_stats">>, {_, _, _, _, _} = Unpacked) ->
+ {ok, pack_stats(Unpacked)};
+finalize(<<"_stats">>, {Packed}) ->
+ % Legacy code path before we had the finalize operation
+ {ok, {Packed}};
+finalize(_RedSrc, Reduction) ->
+ {ok, Reduction}.
rereduce(_Lang, [], _ReducedValues) ->
{ok, []};
@@ -250,11 +251,11 @@ sum_arrays(Else, _) ->
throw_sum_error(Else).
builtin_stats(_, []) ->
- {[{sum,0}, {count,0}, {min,0}, {max,0}, {sumsqr,0}]};
+ {0, 0, 0, 0, 0};
builtin_stats(_, [[_,First]|Rest]) ->
- Unpacked = lists:foldl(fun([_Key, Value], Acc) -> stat_values(Value, Acc) end,
- build_initial_accumulator(First), Rest),
- pack_stats(Unpacked).
+ lists:foldl(fun([_Key, Value], Acc) ->
+ stat_values(Value, Acc)
+ end, build_initial_accumulator(First), Rest).
stat_values(Value, Acc) when is_list(Value), is_list(Acc) ->
lists:zipwith(fun stat_values/2, Value, Acc);
@@ -281,6 +282,8 @@ build_initial_accumulator(L) when is_list(L) ->
[build_initial_accumulator(X) || X <- L];
build_initial_accumulator(X) when is_number(X) ->
{X, 1, X, X, X*X};
+build_initial_accumulator({_, _, _, _, _} = AlreadyUnpacked) ->
+ AlreadyUnpacked;
build_initial_accumulator({Props}) ->
unpack_stats({Props});
build_initial_accumulator(Else) ->
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 4d8d0e9..69f4290 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -230,7 +230,7 @@ get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined ->
end, Counters0, Records),
Wrapped = [[V] || #view_row{value=V} <- Records],
{ok, [Reduced]} = couch_query_servers:rereduce(Lang, [RedSrc], Wrapped),
- {ok, [Finalized]} = couch_query_servers:finalize([Reduced]),
+ {ok, Finalized} = couch_query_servers:finalize(RedSrc, Reduced),
NewSt = St#collector{keys=RestKeys, rows=NewRowDict, counters=Counters},
{#view_row{key=Key, id=reduced, value=Finalized}, NewSt};
error ->
--
To stop receiving notification emails like this one, please contact
kocolosk@apache.org.
[couchdb] 03/03: Ignore trailing characters in a builtin reduce
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 398ac18e8f377b5b3f11ad739bad05b53594a862
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Wed Jun 6 23:11:49 2018 -0400
Ignore trailing characters in a builtin reduce
What a kooky idea, but I guess we're committed to it.
---
src/couch/src/couch_query_servers.erl | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
index fe04533..de8ef1e 100644
--- a/src/couch/src/couch_query_servers.erl
+++ b/src/couch/src/couch_query_servers.erl
@@ -87,12 +87,12 @@ group_reductions_results(List) ->
[Heads | group_reductions_results(Tails)]
end.
-finalize(<<"_approx_count_distinct">>, Reduction) ->
+finalize(<<"_approx_count_distinct",_/binary>>, Reduction) ->
true = hyper:is_hyper(Reduction),
{ok, round(hyper:card(Reduction))};
-finalize(<<"_stats">>, {_, _, _, _, _} = Unpacked) ->
+finalize(<<"_stats",_/binary>>, {_, _, _, _, _} = Unpacked) ->
{ok, pack_stats(Unpacked)};
-finalize(<<"_stats">>, {Packed}) ->
+finalize(<<"_stats",_/binary>>, {Packed}) ->
% Legacy code path before we had the finalize operation
{ok, {Packed}};
finalize(_RedSrc, Reduction) ->
--
To stop receiving notification emails like this one, please contact
kocolosk@apache.org.