You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2015/06/30 17:39:47 UTC
[1/5] couch-mrview commit: updated refs/heads/master to 19737b7
Repository: couchdb-couch-mrview
Updated Branches:
refs/heads/master 1dc8d5252 -> 19737b758
Rework to support equivalent-but-not-equal keys
The range collation test would fail when presented with a set of keys
that compare equal according to ICU but not according to Erlang's =:=
operator.
Other todos here include testing the raw collator and ensuring that all
tests use the correct collator (most of the other ones are also
technically incorrect but we just don't see it with the choices of data
involved). Taking care of this incrementally rather than rewriting the
whole module.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/2dc76491
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/2dc76491
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/2dc76491
Branch: refs/heads/master
Commit: 2dc7649184bf9146d43e07ed89b455d7aa8c0932
Parents: 8e8420b
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jun 29 17:41:32 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jun 29 21:43:48 2015 -0400
----------------------------------------------------------------------
test/couch_mrview_collation_tests.erl | 49 +++++++++++++++++++-----------
1 file changed, 31 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/2dc76491/test/couch_mrview_collation_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_collation_tests.erl b/test/couch_mrview_collation_tests.erl
index df06ad6..4a03359 100644
--- a/test/couch_mrview_collation_tests.erl
+++ b/test/couch_mrview_collation_tests.erl
@@ -34,6 +34,12 @@
<<"ba">>,
<<"bb">>,
+ % U+200B is a zero-width space, which will be ignored by ICU but will cause
+ % the raw collator to treat these as three distinct keys
+ <<"c">>,
+ unicode:characters_to_binary([$c, 16#200B]),
+ unicode:characters_to_binary([$c, 16#200B, 16#200B]),
+
[<<"a">>],
[<<"b">>],
[<<"b">>, <<"c">>],
@@ -73,7 +79,7 @@ collation_test_() ->
[
fun should_collate_fwd/1,
fun should_collate_rev/1,
- fun should_collate_range/1,
+ fun should_collate_range_/1,
fun should_collate_with_inclusive_end_fwd/1,
fun should_collate_with_inclusive_end_rev/1,
fun should_collate_without_inclusive_end_fwd/1,
@@ -95,23 +101,30 @@ should_collate_rev(Db) ->
Expect = [{meta, [{total, length(?VALUES)}, {offset, 0}]}] ++ lists:reverse(rows()),
?_assertEquiv(Expect, Results).
-should_collate_range(Db) ->
- ?_assertNot(
- begin
- {_, Error} = lists:foldl(fun(V, {Count, Error}) ->
- {ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]),
- Id = list_to_binary(integer_to_list(Count)),
- Expect = [
- {meta, [{total, length(?VALUES)}, {offset, Count}]},
- {row, [{id, Id}, {key, V}, {value, 0}]}
- ],
- case Results == Expect of
- true -> {Count+1, Error};
- _ -> {Count+1, true}
- end
- end, {0, false}, ?VALUES),
- Error
- end).
+should_collate_range_(Db) ->
+ Index = lists:zip(lists:seq(0, length(?VALUES)-1), ?VALUES),
+ lists:map(fun(V) ->
+ {ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]),
+ Expect = [
+ {meta, [{total, length(?VALUES)}, find_offset(Index, V)]} |
+ find_matching_rows(Index, V)
+ ],
+ ?_assertEquiv(Expect, Results)
+ end, ?VALUES).
+
+find_offset(Index, Value) ->
+ [{Offset, _} | _] = lists:dropwhile(fun({_, V}) ->
+ couch_ejson_compare:less(Value, V) =/= 0
+ end, Index),
+ {offset, Offset}.
+
+find_matching_rows(Index, Value) ->
+ Matches = lists:filter(fun({_, V}) ->
+ couch_ejson_compare:less(Value, V) =:= 0
+ end, Index),
+ lists:map(fun({Id, V}) ->
+ {row, [{id, list_to_binary(integer_to_list(Id))}, {key, V}, {value, 0}]}
+ end, Matches).
should_collate_with_inclusive_end_fwd(Db) ->
Opts = [{end_key, <<"b">>}, {inclusive_end, true}],
[4/5] couch-mrview commit: updated refs/heads/master to 19737b7
Posted by ko...@apache.org.
Use proper collation fun for reduce grouping
We had been using raw collation for grouping keys in a MR view
regardless of the overall collation setting for the view. This patch
defers the construction of the grouping function to the btree code,
where we have access to the correct sorting function.
COUCHDB-1805
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/ba84fb80
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/ba84fb80
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/ba84fb80
Branch: refs/heads/master
Commit: ba84fb80900e9a86ffa7a05dfcac3c092f2920e1
Parents: 14f5bba
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Fri Jun 26 20:34:28 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jun 29 22:45:41 2015 -0400
----------------------------------------------------------------------
src/couch_mrview.erl | 16 ++--------------
1 file changed, 2 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/ba84fb80/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index f4ba6e1..5583f1f 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -440,8 +440,8 @@ red_fold(Db, {_Nth, _Lang, View}=RedView, Args, Callback, UAcc) ->
update_seq=View#mrview.update_seq,
args=Args
},
- GroupFun = group_rows_fun(Args#mrargs.group_level),
- OptList = couch_mrview_util:key_opts(Args, [{key_group_fun, GroupFun}]),
+ Grouping = {key_group_level, Args#mrargs.group_level},
+ OptList = couch_mrview_util:key_opts(Args, [Grouping]),
Acc2 = lists:foldl(fun(Opts, Acc0) ->
{ok, Acc1} =
couch_mrview_util:fold_reduce(RedView, fun red_fold/3, Acc0, Opts),
@@ -530,18 +530,6 @@ make_meta(Args, UpdateSeq, Base) ->
end.
-group_rows_fun(exact) ->
- fun({Key1,_}, {Key2,_}) -> Key1 == Key2 end;
-group_rows_fun(0) ->
- fun(_A, _B) -> true end;
-group_rows_fun(GroupLevel) when is_integer(GroupLevel) ->
- fun({[_|_] = Key1,_}, {[_|_] = Key2,_}) ->
- lists:sublist(Key1, GroupLevel) == lists:sublist(Key2, GroupLevel);
- ({Key1,_}, {Key2,_}) ->
- Key1 == Key2
- end.
-
-
default_cb(complete, Acc) ->
{ok, lists:reverse(Acc)};
default_cb({final, Info}, []) ->
[5/5] couch-mrview commit: updated refs/heads/master to 19737b7
Posted by ko...@apache.org.
Merge branch '1805-respect-collation-setting-reduce-group'
COUCHDB-1805
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/19737b75
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/19737b75
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/19737b75
Branch: refs/heads/master
Commit: 19737b758363b56ba14515b677cbe485b773c3bf
Parents: 1dc8d52 ba84fb8
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Tue Jun 30 11:39:23 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Tue Jun 30 11:39:23 2015 -0400
----------------------------------------------------------------------
src/couch_mrview.erl | 16 +-----
src/couch_mrview_test_util.erl | 11 ++++-
test/couch_mrview_collation_tests.erl | 78 +++++++++++++++++++-----------
test/couch_mrview_red_views_tests.erl | 2 +-
4 files changed, 62 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
[2/5] couch-mrview commit: updated refs/heads/master to 19737b7
Posted by ko...@apache.org.
Use the ?assertEquiv macro for better reporting
Also dynamically compute the value for the `total` field to allow us to
add additional test data over time.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/8e8420b8
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/8e8420b8
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/8e8420b8
Branch: refs/heads/master
Commit: 8e8420b8873b5b79b07a3076923539a8f5532d14
Parents: 1dc8d52
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jun 29 16:33:53 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jun 29 21:43:48 2015 -0400
----------------------------------------------------------------------
test/couch_mrview_collation_tests.erl | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/8e8420b8/test/couch_mrview_collation_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_collation_tests.erl b/test/couch_mrview_collation_tests.erl
index ccfe5ea..df06ad6 100644
--- a/test/couch_mrview_collation_tests.erl
+++ b/test/couch_mrview_collation_tests.erl
@@ -87,17 +87,13 @@ collation_test_() ->
should_collate_fwd(Db) ->
{ok, Results} = run_query(Db, []),
- Expect = [{meta, [{total, 26}, {offset, 0}]}] ++ rows(),
- %% cannot use _assertEqual since mrview converts
- %% value 3.0 to 3 making assertion fail
- ?_assert(Expect == Results).
+ Expect = [{meta, [{total, length(?VALUES)}, {offset, 0}]}] ++ rows(),
+ ?_assertEquiv(Expect, Results).
should_collate_rev(Db) ->
{ok, Results} = run_query(Db, [{direction, rev}]),
- Expect = [{meta, [{total, 26}, {offset, 0}]}] ++ lists:reverse(rows()),
- %% cannot use _assertEqual since mrview converts
- %% value 3.0 to 3 making assertion fail
- ?_assert(Expect == Results).
+ Expect = [{meta, [{total, length(?VALUES)}, {offset, 0}]}] ++ lists:reverse(rows()),
+ ?_assertEquiv(Expect, Results).
should_collate_range(Db) ->
?_assertNot(
@@ -106,7 +102,7 @@ should_collate_range(Db) ->
{ok, Results} = run_query(Db, [{start_key, V}, {end_key, V}]),
Id = list_to_binary(integer_to_list(Count)),
Expect = [
- {meta, [{total, 26}, {offset, Count}]},
+ {meta, [{total, length(?VALUES)}, {offset, Count}]},
{row, [{id, Id}, {key, V}, {value, 0}]}
],
case Results == Expect of
[3/5] couch-mrview commit: updated refs/heads/master to 19737b7
Posted by ko...@apache.org.
Add a failing test for reduce key grouping
I needed to refactor a few things to support multiple ddocs in the
collation test module.
COUCHDB-1805
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/14f5bba6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/14f5bba6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/14f5bba6
Branch: refs/heads/master
Commit: 14f5bba64b8525f65d70b59e6bf77954fdcded6e
Parents: 2dc7649
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jun 29 21:42:40 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jun 29 22:45:41 2015 -0400
----------------------------------------------------------------------
src/couch_mrview_test_util.erl | 11 ++++++++++-
test/couch_mrview_collation_tests.erl | 15 +++++++++++++--
test/couch_mrview_red_views_tests.erl | 2 +-
3 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/14f5bba6/src/couch_mrview_test_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index 153d8ea..8cebd6d 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -89,7 +89,7 @@ ddoc(map) ->
]});
ddoc(red) ->
couch_doc:from_json_obj({[
- {<<"_id">>, <<"_design/bar">>},
+ {<<"_id">>, <<"_design/red">>},
{<<"views">>, {[
{<<"baz">>, {[
{<<"map">>, <<
@@ -98,6 +98,15 @@ ddoc(red) ->
"}\n"
>>},
{<<"reduce">>, <<"function(keys, vals) {return sum(vals);}">>}
+ ]}},
+ {<<"zing">>, {[
+ {<<"map">>, <<
+ "function(doc) {\n"
+ " if(doc.foo !== undefined)\n"
+ " emit(doc.foo, null);\n"
+ "}"
+ >>},
+ {<<"reduce">>, <<"_count">>}
]}}
]}}
]}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/14f5bba6/test/couch_mrview_collation_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_collation_tests.erl b/test/couch_mrview_collation_tests.erl
index 4a03359..c4a714d 100644
--- a/test/couch_mrview_collation_tests.erl
+++ b/test/couch_mrview_collation_tests.erl
@@ -58,7 +58,8 @@
setup() ->
{ok, Db1} = couch_mrview_test_util:new_db(?tempdb(), map),
- {ok, Db2} = couch_mrview_test_util:save_docs(Db1, make_docs()),
+ Docs = [couch_mrview_test_util:ddoc(red) | make_docs()],
+ {ok, Db2} = couch_mrview_test_util:save_docs(Db1, Docs),
Db2.
teardown(Db) ->
@@ -84,7 +85,8 @@ collation_test_() ->
fun should_collate_with_inclusive_end_rev/1,
fun should_collate_without_inclusive_end_fwd/1,
fun should_collate_without_inclusive_end_rev/1,
- fun should_collate_with_endkey_docid/1
+ fun should_collate_with_endkey_docid/1,
+ fun should_use_collator_for_reduce_grouping/1
]
}
}
@@ -173,6 +175,12 @@ should_collate_with_endkey_docid(Db) ->
?assertEqual(Expect1, Result1)
end).
+should_use_collator_for_reduce_grouping(Db) ->
+ UniqueKeys = lists:usort(fun(A, B) ->
+ not couch_ejson_compare:less_json(B, A)
+ end, ?VALUES),
+ {ok, [{meta,_} | Rows]} = reduce_query(Db, [{group_level, exact}]),
+ ?_assertEqual(length(UniqueKeys), length(Rows)).
make_docs() ->
{Docs, _} = lists:foldl(fun(V, {Docs0, Count}) ->
@@ -194,3 +202,6 @@ rows() ->
run_query(Db, Opts) ->
couch_mrview:query_view(Db, <<"_design/bar">>, <<"zing">>, Opts).
+
+reduce_query(Db, Opts) ->
+ couch_mrview:query_view(Db, <<"_design/red">>, <<"zing">>, Opts).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/14f5bba6/test/couch_mrview_red_views_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_red_views_tests.erl b/test/couch_mrview_red_views_tests.erl
index ae3e454..3100785 100644
--- a/test/couch_mrview_red_views_tests.erl
+++ b/test/couch_mrview_red_views_tests.erl
@@ -92,4 +92,4 @@ should_reduce_with_group_exact(Db) ->
run_query(Db, Opts) ->
- couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).
+ couch_mrview:query_view(Db, <<"_design/red">>, <<"baz">>, Opts).