You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/03/02 17:08:01 UTC
[1/2] couch-mrview commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to e204e51 [Forced Update!]
Repository: couchdb-couch-mrview
Updated Branches:
refs/heads/COUCHDB-3287-pluggable-storage-engines c8376a1fa -> e204e51f7 (forced update)
Fix unit tests in couch_mrview_compactor
This fixes the two tests that are in couch_mrview_compactor.erl. For
some reason they don't always run in CI or even for local `make check`
runs. However when they do run they fail as they don't account for how
couch_index_updater:update/3 works and don't start couch_log.
The theory for why these don't always run is related to how meck loads
modules. I'm told that these should probably be moved to
test/couch_mrview_compactor_tests.erl but then that would remove access
to the recompact/1 function. For now I'll leave them here I guess and if
they do ever run they'll not break the build now.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/e204e51f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/e204e51f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/e204e51f
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: e204e51f72ffcdbdb4aaf66caf25519b9c7a1f37
Parents: 83aabd3
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 8 12:11:47 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Mar 2 11:07:30 2017 -0600
----------------------------------------------------------------------
src/couch_mrview_compactor.erl | 30 +++++++++++++++++++-----------
1 file changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/e204e51f/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 3bd539d..3c30d2f 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -304,13 +304,16 @@ recompact_success_after_progress() ->
?_test(begin
ok = meck:expect(couch_index_updater, update, fun
(Pid, _, #mrst{update_seq=0} = State) ->
- Pid ! {'$gen_cast', {new_state, State#mrst{update_seq=1}}};
- (_, _, State) ->
- exit({updated, self(), State})
+ Pid ! {'$gen_cast', {new_state, State#mrst{update_seq = 1}}},
+ timer:sleep(100),
+ exit({updated, self(), State#mrst{update_seq = 2}})
end),
- State = #mrst{fd=self(), update_seq=0},
- ?assertEqual({ok, State#mrst{update_seq=1}}, recompact(State)),
- meck:unload(couch_index_updater)
+ try
+ State = #mrst{fd=self(), update_seq=0},
+ ?assertEqual({ok, State#mrst{update_seq = 2}}, recompact(State))
+ after
+ meck:unload(couch_index_updater)
+ end
end).
recompact_exceeded_retry_count() ->
@@ -319,11 +322,16 @@ recompact_exceeded_retry_count() ->
fun(_, _, _) ->
exit(error)
end),
- State = #mrst{fd=self(), db_name=foo, idx_name=bar},
- ExpectedError = {exceeded_recompact_retry_count,
- [{db_name, foo}, {idx_name, bar}]},
- ?assertError(ExpectedError, recompact(State)),
- meck:unload(couch_index_updater)
+ ok = meck:expect(couch_log, warning, fun(_, _) -> ok end),
+ try
+ State = #mrst{fd=self(), db_name=foo, idx_name=bar},
+ ExpectedError = {exceeded_recompact_retry_count,
+ [{db_name, foo}, {idx_name, bar}]},
+ ?assertError(ExpectedError, recompact(State))
+ after
+ meck:unload(couch_log),
+ meck:unload(couch_index_updater)
+ end
end).
-endif.
[2/2] couch-mrview commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to e204e51
Posted by da...@apache.org.
Update to use pluggable storage API
COUCHDB-3287
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/83aabd3e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/83aabd3e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/83aabd3e
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 83aabd3ea9d97cd5b500c067a1ec0c1cced8ce7f
Parents: a271447
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 10 16:31:45 2016 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Mar 2 11:07:30 2017 -0600
----------------------------------------------------------------------
src/couch_mrview.erl | 34 +++++++++++++++++++++++++++-------
src/couch_mrview_util.erl | 17 ++++++++++-------
2 files changed, 37 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/83aabd3e/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 0f207fb..c15bbad 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -417,8 +417,18 @@ all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) ->
update_seq=UpdateSeq,
args=Args
},
- [Opts] = couch_mrview_util:all_docs_key_opts(Args),
- {ok, Offset, FinalAcc} = couch_db:enum_docs(Db, fun map_fold/3, Acc, Opts),
+ [Opts1] = couch_mrview_util:all_docs_key_opts(Args),
+ % TODO: This is a terrible hack for now. We'll probably have
+ % to rewrite _all_docs to not be part of mrview and not expect
+ % a btree. For now non-btree's will just have to pass 0 or
+ % some fake reductions to get an offset.
+ Opts2 = [include_reductions | Opts1],
+ FunName = case couch_util:get_value(namespace, Args#mrargs.extra) of
+ <<"_design">> -> fold_design_docs;
+ <<"_local">> -> fold_local_docs;
+ _ -> fold_docs
+ end,
+ {ok, Offset, FinalAcc} = couch_db:FunName(Db, fun map_fold/3, Acc, Opts2),
finish_fold(FinalAcc, [{total, Total}, {offset, Offset}]);
all_docs_fold(Db, #mrargs{direction=Dir, keys=Keys0}=Args, Callback, UAcc) ->
{ok, Info} = couch_db:get_db_info(Db),
@@ -533,15 +543,25 @@ map_fold({{Key, Id}, Val}, _Offset, Acc) ->
user_acc=UAcc1,
last_go=Go
}};
-map_fold({<<"_local/",_/binary>> = DocId, {Rev0, _Body}}, _Offset, #mracc{} = Acc) ->
+map_fold(#doc{id = <<"_local/", _/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
#mracc{
limit=Limit,
callback=Callback,
- user_acc=UAcc0
+ user_acc=UAcc0,
+ args=Args
} = Acc,
- Rev = {0, list_to_binary(integer_to_list(Rev0))},
- Value = {[{rev, couch_doc:rev_to_str(Rev)}]},
- Row = [{id, DocId}, {key, DocId}, {value, Value}],
+ #doc{
+ id = DocId,
+ revs = {Pos, [RevId | _]}
+ } = Doc,
+ Rev = {Pos, RevId},
+ Row = [
+ {id, DocId},
+ {key, DocId},
+ {value, {[{rev, couch_doc:rev_to_str(Rev)}]}}
+ ] ++ if not Args#mrargs.include_docs -> []; true ->
+ [{doc, couch_doc:to_json_obj(Doc, Args#mrargs.doc_options)}]
+ end,
{Go, UAcc1} = Callback({row, Row}, UAcc0),
{Go, Acc#mracc{
limit=Limit-1,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/83aabd3e/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 3830b96..2e8ef4c 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -165,7 +165,7 @@ extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) ->
view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
UpdateSeq = couch_db:get_update_seq(Db),
- PurgeSeq = couch_db:get_purge_seq(Db),
+ {ok, PurgeSeq} = couch_db:get_purge_seq(Db),
#mrst{
seq_indexed=SeqIndexed,
keyseq_indexed=KeySeqIndexed
@@ -199,9 +199,10 @@ view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) ->
init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
+ {ok, PurgeSeq} = couch_db:get_purge_seq(Db),
Header = #mrheader{
seq=0,
- purge_seq=couch_db:get_purge_seq(Db),
+ purge_seq=PurgeSeq,
id_btree_state=nil,
log_btree_state=nil,
view_states=[make_view_state(#mrview{}) || _ <- Views]
@@ -236,7 +237,9 @@ init_state(Db, Fd, State, Header) ->
view_states=ViewStates
} = Header,
- IdBtOpts = [{compression, couch_db:compression(Db)}],
+ IdBtOpts = [
+ {compression, couch_compress:get_compression_method()}
+ ],
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
{ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of
true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
@@ -256,10 +259,10 @@ init_state(Db, Fd, State, Header) ->
views=Views2
}.
-open_view(Db, Fd, Lang, ViewState, View) ->
+open_view(_Db, Fd, Lang, ViewState, View) ->
ReduceFun = make_reduce_fun(Lang, View#mrview.reduce_funs),
LessFun = maybe_define_less_fun(View),
- Compression = couch_db:compression(Db),
+ Compression = couch_compress:get_compression_method(),
BTState = get_key_btree_state(ViewState),
ViewBtOpts = [
{less, LessFun},
@@ -268,7 +271,7 @@ open_view(Db, Fd, Lang, ViewState, View) ->
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
+ BySeqReduceFun = fun couch_bt_engine:seq_tree_reduce/2,
{ok, SeqBtree} = if View#mrview.seq_indexed ->
SeqBTState = get_seq_btree_state(ViewState),
ViewSeqBtOpts = [{reduce, BySeqReduceFun},
@@ -320,7 +323,7 @@ get_row_count(#mrview{btree=Bt}) ->
all_docs_reduce_to_count(Reductions) ->
- Reduce = fun couch_db_updater:btree_by_id_reduce/2,
+ Reduce = fun couch_bt_engine:id_tree_reduce/2,
{Count, _, _} = couch_btree:final_reduce(Reduce, Reductions),
Count.