You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2017/10/31 23:55:58 UTC
[couchdb] 01/01: fix external size bug when seq btree exists
This is an automated email from the ASF dual-hosted git repository.
tonysun83 pushed a commit to branch seqt-external-size-bug
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1b79cd4d98f8c283f578262b92bf2cf002dc8a0b
Author: Tony Sun <to...@gmail.com>
AuthorDate: Thu Sep 28 15:40:17 2017 -0700
fix external size bug when seq btree exists
When users specify "options": {"seq_indexed": true}, we will run
reduced_external_size/1 on seq btrees. The reduce will return an
integer because the reduce function is:
couch_db_updater:btree_by_seq_reduce/2.
This leads to a bad match error. Thanks @jiangphcn for finding this.
---
src/couch_mrview/src/couch_mrview.erl | 18 +++++++-----
src/couch_mrview/src/couch_mrview_util.erl | 34 ++++++++++++++++++++--
.../test/couch_mrview_map_views_tests.erl | 23 ++++++++++++++-
3 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 07e3668..83a03ff 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -314,15 +314,17 @@ count_view_changes_since(Db, DDoc, VName, SinceSeq, Options) ->
true -> View#mrview.key_byseq_btree;
_ -> View#mrview.seq_btree
end,
+ RedFun = fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(Btree, PartialReds)}
+ end,
lists:foldl(fun(Opts, Acc0) ->
- {ok, N} = couch_btree:fold_reduce(
- Btree, fun(_SeqStart, PartialReds, 0) ->
- {ok, couch_btree:final_reduce(
- Btree, PartialReds)}
- end,
- 0, Opts),
- Acc0 + N
- end, 0, OptList);
+ case couch_btree:fold_reduce(Btree, RedFun, 0, Opts) of
+ {ok, N} when is_integer(N) ->
+ Acc0 + N;
+ {ok, N} when is_tuple(N) ->
+ Acc0 + element(1, N)
+ end
+ end, 0, OptList);
_ ->
{error, seqs_not_indexed}
end.
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index 0d58e4f..d26df94 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -299,7 +299,7 @@ open_view(Db, Fd, Lang, ViewState, View) ->
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
+ BySeqReduceFun = make_seq_reduce_fun(),
{ok, SeqBtree} = if View#mrview.seq_indexed ->
SeqBTState = get_seq_btree_state(ViewState),
ViewSeqBtOpts = [{reduce, BySeqReduceFun},
@@ -804,16 +804,24 @@ reduced_external_size(Tree) ->
end.
+reduced_seq_external_size(Tree) ->
+ case couch_btree:full_reduce(Tree) of
+ {ok, {_, Size}} -> Size;
+ % return 0 for older versions that only returned number of docs
+ {ok, NumDocs} when is_integer(NumDocs) -> 0
+ end.
+
+
calculate_external_size(Views) ->
SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
Size0 = sum_btree_sizes(Acc, reduced_external_size(Bt)),
Size1 = case SBt of
nil -> Size0;
- _ -> sum_btree_sizes(Size0, reduced_external_size(SBt))
+ _ -> sum_btree_sizes(Size0, reduced_seq_external_size(SBt))
end,
case KSBt of
nil -> Size1;
- _ -> sum_btree_sizes(Size1, reduced_external_size(KSBt))
+ _ -> sum_btree_sizes(Size1, reduced_seq_external_size(KSBt))
end
end,
{ok, lists:foldl(SumFun, 0, Views)}.
@@ -1044,6 +1052,10 @@ get_user_reds(Reduction) ->
element(2, Reduction).
+% This is for backwards compatibility for seq btree reduces
+get_external_size_reds(Reduction) when is_integer(Reduction) ->
+ 0;
+
get_external_size_reds(Reduction) when tuple_size(Reduction) == 2 ->
0;
@@ -1072,6 +1084,22 @@ make_reduce_fun(Lang, ReduceFuns) ->
{Counts, Result, ExternalSize}
end.
+make_seq_reduce_fun() ->
+ fun
+ (reduce, KVs0) ->
+ KVs = detuple_kvs(expand_dups(KVs0, []), []),
+ NumDocs = length(KVs),
+ ExternalSize = kv_external_size(KVs, NumDocs),
+ {NumDocs, ExternalSize};
+ (rereduce, Reds) ->
+ ExtractFun = fun(Red, {NumDocsAcc0, ExtAcc0}) ->
+ NumDocsAcc = NumDocsAcc0 + get_count(Red),
+ ExtAcc = ExtAcc0 + get_external_size_reds(Red),
+ {NumDocsAcc, ExtAcc}
+ end,
+ lists:foldl(ExtractFun, {0, 0}, Reds)
+ end.
+
maybe_define_less_fun(#mrview{options = Options}) ->
case couch_util:get_value(<<"collation">>, Options) of
diff --git a/src/couch_mrview/test/couch_mrview_map_views_tests.erl b/src/couch_mrview/test/couch_mrview_map_views_tests.erl
index 229af18..805dc6c 100644
--- a/src/couch_mrview/test/couch_mrview_map_views_tests.erl
+++ b/src/couch_mrview/test/couch_mrview_map_views_tests.erl
@@ -42,7 +42,8 @@ map_views_test_() ->
fun should_map_with_range/1,
fun should_map_with_limit_and_skip/1,
fun should_map_with_include_docs/1,
- fun should_map_empty_views/1
+ fun should_map_empty_views/1,
+ fun should_give_ext_size_seq_indexed_test/1
]
}
}
@@ -118,6 +119,26 @@ should_map_empty_views(Db) ->
]},
?_assertEqual(Expect, Result).
+should_give_ext_size_seq_indexed_test(Db) ->
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/seqdoc">>},
+ {<<"options">>, {[{<<"seq_indexed">>, true}]}},
+ {<<"views">>, {[
+ {<<"view1">>, {[
+ {<<"map">>, <<"function(doc){emit(doc._id, doc._id);}">>}
+ ]}}
+ ]}
+ }
+ ]}),
+ {ok, _} = couch_db:update_doc(Db, DDoc, []),
+ {ok, Db1} = couch_db:open_int(couch_db:name(Db), []),
+ {ok, DDoc1} = couch_db:open_doc(Db1, <<"_design/seqdoc">>, [ejson_body]),
+ couch_mrview:query_view(Db1, DDoc1, <<"view1">>, [{update, true}]),
+ {ok, Info} = couch_mrview:get_info(Db1, DDoc),
+ Size = couch_util:get_nested_json_value({Info}, [sizes, external]),
+ ok = couch_db:close(Db1),
+ ?_assert(is_number(Size)).
+
run_query(Db, Opts) ->
couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.