You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2019/09/18 07:44:00 UTC
[couchdb] branch prototype/builtin-reduce updated (2cd1db7 ->
744625c)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch prototype/builtin-reduce
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 2cd1db7 group=true and group_levels working
new 22fc257 stream large
new 744625c return group over exact for set group_level
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch_views/include/couch_views.hrl | 4 +-
src/couch_views/src/couch_views_fdb.erl | 36 ++--
src/couch_views/src/couch_views_reader.erl | 4 +-
.../test/exunit/couch_views_reduce_test.exs | 217 +++++++++++++--------
4 files changed, 159 insertions(+), 102 deletions(-)
[couchdb] 01/02: stream large
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/builtin-reduce
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 22fc2578947d81f621fc72407662609510514c5e
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Sep 12 17:36:11 2019 +0200
stream large
---
src/couch_views/src/couch_views_reader.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 59c648f..359193a 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -138,8 +138,8 @@ reduce_mrargs_to_fdb_options(Args) ->
[
{dir, Direction},
%% {limit, Limit * 2 + Skip * 2},
-%% {streaming_mode, stream_large}
- {streaming_mode, want_all}
+ {streaming_mode, large}
+%% {streaming_mode, want_all}
] ++ GroupLevelEnd.
%% ] ++ StartKeyOpts ++ EndKeyOpts.
[couchdb] 02/02: return group over exact for set group_level
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/builtin-reduce
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 744625c3e0d46f1b7a348fbbaff46a2231256183
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Sep 18 09:43:43 2019 +0200
return group over exact for set group_level
---
src/couch_views/include/couch_views.hrl | 4 +-
src/couch_views/src/couch_views_fdb.erl | 36 ++--
.../test/exunit/couch_views_reduce_test.exs | 217 +++++++++++++--------
3 files changed, 157 insertions(+), 100 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 6b029f7..aa272ce 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -23,8 +23,8 @@
-define(VIEW_ROW_KEY, 0).
-define(VIEW_ROW_VALUE, 1).
--define(VIEW_REDUCE_EXACT, 0).
--define(VIEW_REDUCE_GROUP, 1).
+-define(VIEW_REDUCE_GROUP, 0).
+-define(VIEW_REDUCE_EXACT, 1).
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 7bfcac0..2eff4bd 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -169,7 +169,7 @@ fold_reduce_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
acc => Acc0,
group_level => GroupLevel,
group => Group1,
- prev_exact_key => undefined,
+ prev_group_key => undefined,
reduce_type => undefined,
next => key,
@@ -208,7 +208,7 @@ reduce_fold_fwd({RowKey, EncodedValue}, #{next := value} = Acc) ->
reduce_type := ReduceType,
callback := UserCallback,
acc := UserAcc0,
- prev_exact_key := PrevExactKey
+ prev_group_key := PrevGroupKey
} = Acc,
@@ -222,14 +222,14 @@ reduce_fold_fwd({RowKey, EncodedValue}, #{next := value} = Acc) ->
Value = ?bin2uint(EncodedValue),
io:format("FWD VAL ~p ~p ~p ~p ~n", [Key, RowGroupLevel, Value, ReduceType]),
io:format("GROUP SETTINGS ~p ~p ~n", [Group, GroupLevel]),
- UserAcc1 = case should_return_row(PrevExactKey, Key, Group, GroupLevel, RowGroupLevel, ReduceType) of
+ UserAcc1 = case should_return_row(PrevGroupKey, Key, Group, GroupLevel, RowGroupLevel, ReduceType) of
true ->
UserCallback(Key, Value, UserAcc0);
false ->
UserAcc0
end,
- PrevExactKey1 = maybe_update_prev_exact_key(PrevExactKey, Key, ReduceType),
+ PrevGroupKey1 = maybe_update_prev_group_key(PrevGroupKey, Key, ReduceType),
Acc#{
next := key,
@@ -237,24 +237,24 @@ reduce_fold_fwd({RowKey, EncodedValue}, #{next := value} = Acc) ->
sort_key := undefined,
reduce_type := undefined,
acc := UserAcc1,
- prev_exact_key := PrevExactKey1
+ prev_group_key := PrevGroupKey1
}.
-should_return_row(_PrevExactKey, _CurrentKey, exact, _GroupLevel,
- _RowGroupLevel, ?VIEW_REDUCE_EXACT) ->
- true;
-
-should_return_row(_PrevExactKey, _CurrentKey, exact, _GroupLevel,
+should_return_row(_PrevGroupKey, _CurrentKey, exact, _GroupLevel,
_RowGroupLevel, ?VIEW_REDUCE_GROUP) ->
false;
-should_return_row(_PrevExactKey, _CurrentKey, _Group, GroupLevel,
- RowGroupLevel, ?VIEW_REDUCE_EXACT) when RowGroupLevel =< GroupLevel ->
+should_return_row(_PrevGroupKey, _CurrentKey, exact, _GroupLevel,
+ _RowGroupLevel, ?VIEW_REDUCE_EXACT) ->
true;
-should_return_row(PrevExactKey, PrevExactKey, _Group, GroupLevel, GroupLevel,
- ?VIEW_REDUCE_GROUP) ->
+should_return_row(_PrevGroupKey, _CurrentKey, _Group, GroupLevel,
+ RowGroupLevel, ?VIEW_REDUCE_EXACT) when RowGroupLevel < GroupLevel ->
+ true;
+
+should_return_row(PrevGroupKey, PrevGroupKey, _Group, GroupLevel, GroupLevel,
+ ?VIEW_REDUCE_EXACT) ->
false;
should_return_row(_PrevExactKey, _CurrentKey, _Group, GroupLevel, GroupLevel,
@@ -265,11 +265,11 @@ should_return_row(_, _, _, _, _, _) ->
false.
-maybe_update_prev_exact_key(PrevExactKey, _NewKey, ?VIEW_REDUCE_GROUP) ->
- PrevExactKey;
+maybe_update_prev_group_key(_PrevGroupKey, NewKey, ?VIEW_REDUCE_GROUP) ->
+ NewKey;
-maybe_update_prev_exact_key(_PrevExactKey, NewKey, ?VIEW_REDUCE_EXACT) ->
- NewKey.
+maybe_update_prev_group_key(PrevGroupKey, _NewKey, ?VIEW_REDUCE_EXACT) ->
+ PrevGroupKey.
write_doc(TxDb, Sig, _ViewIds, #{deleted := true} = Doc) ->
diff --git a/src/couch_views/test/exunit/couch_views_reduce_test.exs b/src/couch_views/test/exunit/couch_views_reduce_test.exs
index 48887d0..f6efc0c 100644
--- a/src/couch_views/test/exunit/couch_views_reduce_test.exs
+++ b/src/couch_views/test/exunit/couch_views_reduce_test.exs
@@ -41,94 +41,120 @@ defmodule CouchViewsReduceTest do
end
test "group=true count reduce", context do
- args = %{
- :reduce => true,
- :group => true
-# :limit => 9
- }
-
- {:ok, res} = run_query(context, args)
- IO.inspect(res, label: "OUT")
-
- assert res == [
- {:row, [key: 1, value: 2]},
- {:row, [key: 2, value: 2]},
- {:row, [key: 3, value: 2]},
- {:row, [key: [1, 1], value: 1]},
- {:row, [key: [1, 2, 6], value: 1]},
- {:row, [key: [2, 1], value: 1]},
- {:row, [key: [2, 3, 6], value: 1]},
- {:row, [key: [3, 1], value: 1]},
- {:row, [key: [3, 4, 5], value: 1]}
- ]
- end
+ args = %{
+ :reduce => true,
+ :group => true
+ # :limit => 9
+ }
- test "group=1 count reduce", context do
- args = %{
- :reduce => true,
- :group_level => 1
-# :limit => 6
- }
-
- {:ok, res} = run_query(context, args)
- IO.inspect(res, label: "OUT")
-
- assert res == [
- {:row, [key: 1, value: 2]},
- {:row, [key: 2, value: 2]},
- {:row, [key: 3, value: 2]},
- {:row, [key: [1], value: 2]},
- {:row, [key: [2], value: 2]},
- {:row, [key: [3], value: 2]}
- ]
+ {:ok, res} = run_query(context, args, "baz")
+ IO.inspect(res, label: "OUT")
+
+ assert res == [
+ {:row, [key: 1, value: 2]},
+ {:row, [key: 2, value: 2]},
+ {:row, [key: 3, value: 2]},
+ {:row, [key: [1, 1], value: 1]},
+ {:row, [key: [1, 1, 5], value: 1]},
+ {:row, [key: [1, 2, 6], value: 1]},
+ {:row, [key: [2, 1], value: 1]},
+ {:row, [key: [2, 3, 6], value: 1]},
+ {:row, [key: [3, 1], value: 1]},
+ {:row, [key: [3, 1, 5], value: 1]},
+ {:row, [key: [3, 4, 5], value: 1]}
+ ]
end
+# test "group=1 count reduce", context do
+# args = %{
+# :reduce => true,
+# :group_level => 1
+# # :limit => 6
+# }
+#
+# {:ok, res} = run_query(context, args, "baz")
+# IO.inspect(res, label: "OUT")
+#
+# assert res == [
+# {:row, [key: 1, value: 2]},
+# {:row, [key: 2, value: 2]},
+# {:row, [key: 3, value: 2]},
+# {:row, [key: [1], value: 2]},
+# {:row, [key: [2], value: 2]},
+# {:row, [key: [3], value: 2]}
+# ]
+# end
+#
test "group=2 count reduce", context do
- args = %{
- :reduce => true,
- :group_level => 2
-# :limit => 9
- }
-
- {:ok, res} = run_query(context, args)
- IO.inspect(res, label: "OUT")
-
- assert res == [
- {:row, [key: 1, value: 2]},
- {:row, [key: 2, value: 2]},
- {:row, [key: 3, value: 2]},
- {:row, [key: [1, 1], value: 1]},
- {:row, [key: [1, 2], value: 1]},
- {:row, [key: [2, 1], value: 1]},
- {:row, [key: [2, 3], value: 1]},
- {:row, [key: [3, 1], value: 1]},
- {:row, [key: [3, 4], value: 1]}
- ]
- end
+ args = %{
+ :reduce => true,
+ :group_level => 2,
+ :limit => 9
+ }
- test "group=2 count reduce with limit = 3", context do
- args = %{
- :reduce => true,
- :group_level => 2,
- :limit => 4
- }
-
- {:ok, res} = run_query(context, args)
- IO.inspect(res, label: "OUT")
-
- assert res == [
- {:row, [key: 1, value: 2]},
- {:row, [key: 2, value: 2]},
- {:row, [key: 3, value: 2]},
- {:row, [key: [1, 1], value: 1]}
- ]
+ {:ok, res} = run_query(context, args, "baz")
+ IO.inspect(res, label: "OUT")
+
+ assert res == [
+ {:row, [key: 1, value: 2]},
+ {:row, [key: 2, value: 2]},
+ {:row, [key: 3, value: 2]},
+ {:row, [key: [1, 1], value: 2]},
+ {:row, [key: [1, 2], value: 1]},
+ {:row, [key: [2, 1], value: 1]},
+ {:row, [key: [2, 3], value: 1]},
+ {:row, [key: [3, 1], value: 2]},
+ {:row, [key: [3, 4], value: 1]}
+ ]
end
-
- defp run_query(context, args) do
+#
+# test "group=2 count reduce with limit = 3", context do
+# args = %{
+# :reduce => true,
+# :group_level => 2,
+# :limit => 4
+# }
+#
+# {:ok, res} = run_query(context, args, "baz")
+# IO.inspect(res, label: "OUT")
+#
+# assert res == [
+# {:row, [key: 1, value: 2]},
+# {:row, [key: 2, value: 2]},
+# {:row, [key: 3, value: 2]},
+# {:row, [key: [1, 1], value: 1]}
+# ]
+# end
+#
+# # [
+# # row: [key: [2019, 1, 2], value: 1],
+# # row: [key: [2019, 1, 4], value: 1],
+# # row: [key: [2019, 2, 1], value: 1],
+# # row: [key: [2019, 2, 3], value: 1]
+# # ]
+#
+# test "group=2 count reduce with startkey", context do
+# args = %{
+# # :reduce => true,
+# # :group_level => 2,
+# :start_key => [2019, 1, 4]
+# # :limit => 4
+# }
+#
+# {:ok, res} = run_query(context, args, "boom")
+# IO.inspect(res, label: "OUT")
+#
+# assert res == [
+# {:row, [key: [2019, 1], value: 1]},
+# {:row, [key: [2019, 2], value: 2]}
+# ]
+# end
+
+ defp run_query(context, args, view) do
db = context[:db]
ddoc = context[:ddoc]
- :couch_views.query(db, ddoc, "baz", &__MODULE__.default_cb/2, [], args)
+ :couch_views.query(db, ddoc, view, &__MODULE__.default_cb/2, [], args)
end
def default_cb(:complete, acc) do
@@ -190,13 +216,44 @@ defmodule CouchViewsReduceTest do
emit(doc.value, doc.value);
emit([doc.value, 1], doc.value);
emit([doc.value, doc.value + 1, doc.group.length], doc.value);
+
+ if (doc.value === 3) {
+ emit([1, 1, 5], 1);
+ emit([doc.value, 1, 5], 1);
+ }
}
"""},
{"reduce", "_count"}
]}},
{"boom",
{[
- {"map", "function(doc) {emit([doc._id, doc.value], doc.value);}"},
+ {"map",
+ """
+ function(doc) {
+ var month = 1;
+ if (doc.value % 2) {
+ month = 2;
+ }
+ emit([2019, month, doc.value], doc.value);
+ }
+ """},
+ {"reduce", "_count"}
+ ]}},
+ {"max",
+ {[
+ {"map",
+ """
+ function(doc) {
+ emit(doc.value, doc.value);
+ emit(doc.value, doc.value);
+ emit([doc.value, 1], doc.value);
+ emit([doc.value, doc.value + 1, doc.group.length], doc.value);
+
+ if (doc.value === 3) {
+ emit([doc.value, 1, 5], 1);
+ }
+ }
+ """},
{"reduce", "_count"}
]}}
]}}