You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by bb...@apache.org on 2014/10/31 20:53:16 UTC
[01/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Repository: couchdb-couch-mrview
Updated Branches:
refs/heads/master 1a6bd404e -> 28e51f3f8
add couch_mrview:view_changes_since/{6,7} function
This function add the possibility to get changes in a view since the
last upddated sequence. You can all changes in a view since a sequence
or all changes for a key or a range in a view.
The following new secondaries are created to allows this feature:
- a generic log index to log the latest changes in views for a docid :
{DocId, [{ViewId, {Key, Seq, OP}}]} where OP can be del or add. This
index allows us to mark a key as removed if needed. It will be useful
later to help us to chain map/reduces operations or such things.
- a seq index associated to a view id : {ViewId, [{Seq, Key}, {DocId, Val}]} to look for all changes in a view
- an index indexing keys by seq: {ViewId, [{[Key, Seq], DocId}, Val}]},
to looks for changes associated to a key or a ranhge
Note: all deleted keys are marked as deleted in the log index and their
value is {[{<<"_removed">>, true}]}.
To start to index changes you need to pass the options {seq_indexed:
true} to the design document.
Caveat: when the changes are indexed the size of the index is significantly higher.
Example of usage: https://www.friendpaste.com/5Y6gihQReaxd8ERqbDom3y
Conflicts:
src/couch_mrview_updater.erl
src/couch_mrview_util.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/44c07c30
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/44c07c30
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/44c07c30
Branch: refs/heads/master
Commit: 44c07c30ad2b39a528b9e3aa5aaa4ea1bc07088f
Parents: 1a6bd40
Author: benoitc <be...@apache.org>
Authored: Sun Jan 26 23:56:09 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:30 2014 -0700
----------------------------------------------------------------------
include/couch_mrview.hrl | 7 +-
src/couch_mrview.erl | 48 ++++++++++
src/couch_mrview_index.erl | 7 +-
src/couch_mrview_updater.erl | 184 +++++++++++++++++++++++++++++--------
src/couch_mrview_util.erl | 186 ++++++++++++++++++++++++++++++++------
5 files changed, 364 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/44c07c30/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index 36c35d6..f1911db 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -18,12 +18,13 @@
idx_name,
language,
design_opts=[],
+ seq_indexed=false,
lib,
views,
id_btree=nil,
+ log_btree=nil,
update_seq=0,
purge_seq=0,
-
first_build,
partial_resp_pid,
doc_acc,
@@ -41,6 +42,9 @@
reduce_funs=[],
def,
btree=nil,
+ seq_btree=nil,
+ key_byseq_btree=nil,
+ seq_indexed=false,
options=[]
}).
@@ -49,6 +53,7 @@
seq=0,
purge_seq=0,
id_btree_state=nil,
+ log_btree_state=nil,
view_states=nil
}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/44c07c30/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 047bc00..afb6f1e 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -15,6 +15,7 @@
-export([validate/2]).
-export([query_all_docs/2, query_all_docs/4]).
-export([query_view/3, query_view/4, query_view/6]).
+-export([view_changes_since/6, view_changes_since/7]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
-export([compact/2, compact/3, cancel_compaction/2]).
@@ -135,6 +136,30 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) ->
erlang:demonitor(Ref, [flush])
end.
+view_changes_since(Db, DDoc, VName, StartSeq, Fun, Acc) ->
+ view_changes_since(Db, DDoc, VName, StartSeq, Fun, [], Acc).
+
+view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
+ Args0 = make_view_changes_args(Options),
+ {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+ Args0),
+ case View#mrview.seq_indexed of
+ true ->
+ OptList = make_view_changes_opts(StartSeq, Options, Args),
+ Btree = case is_key_byseq(Options) of
+ true -> View#mrview.key_byseq_btree;
+ _ -> View#mrview.seq_btree
+ end,
+ io:format("opt list ~p~n", [OptList]),
+ AccOut = lists:foldl(fun(Opts, Acc0) ->
+ {ok, _R, A} = couch_mrview_util:fold_changes(
+ Btree, Fun, Acc0, Opts),
+ A
+ end, Acc, OptList),
+ {ok, AccOut};
+ _ ->
+ {error, seqs_not_indexed}
+ end.
get_info(Db, DDocId) when is_binary(DDocId) ->
DbName = mem3:dbname(Db#db.name),
@@ -447,3 +472,26 @@ lookup_index(Key) ->
record_info(fields, mrargs), lists:seq(2, record_info(size, mrargs))
),
couch_util:get_value(Key, Index).
+
+
+is_key_byseq(Options) ->
+ lists:any(fun({K, _}) ->
+ lists:member(K, [start_key, end_key, start_key_docid,
+ end_key_docid, keys])
+ end, Options).
+
+make_view_changes_args(Options) ->
+ case is_key_byseq(Options) of
+ true ->
+ to_mrargs(Options);
+ false ->
+ #mrargs{}
+ end.
+
+make_view_changes_opts(StartSeq, Options, Args) ->
+ case is_key_byseq(Options) of
+ true ->
+ couch_mrview_util:changes_key_opts(StartSeq, Args);
+ false ->
+ [[{start_key, {StartSeq+1, <<>>}}] ++ Options]
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/44c07c30/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 2d6ccae..0b9b236 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -51,14 +51,17 @@ get(Property, State) ->
#mrst{
fd = Fd,
sig = Sig,
- id_btree = Btree,
+ id_btree = IdBtree,
+ log_btree = LogBtree,
language = Lang,
update_seq = UpdateSeq,
purge_seq = PurgeSeq,
views = Views
} = State,
{ok, FileSize} = couch_file:bytes(Fd),
- {ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
+ {ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
+ LogBtree,
+ Views),
ActiveSize = ExternalSize + couch_btree:size(Btree),
{ok, [
{signature, list_to_binary(couch_index_util:hexsig(Sig))},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/44c07c30/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 1525a4c..eb06c92 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -141,12 +141,12 @@ map_docs(Parent, State0) ->
({nil, Seq, _}, {SeqAcc, Results}) ->
{erlang:max(Seq, SeqAcc), Results};
({Id, Seq, deleted}, {SeqAcc, Results}) ->
- {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, []} | Results]};
({Id, Seq, Doc}, {SeqAcc, Results}) ->
couch_stats:increment_counter([couchdb, mrview, map_docs],
1),
{ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
- {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Res} | Results]}
end,
FoldFun = fun(Docs, Acc) ->
update_task(length(Docs)),
@@ -162,8 +162,8 @@ write_results(Parent, State) ->
case accumulate_writes(State, State#mrst.write_queue, nil) of
stop ->
Parent ! {new_state, State};
- {Go, {Seq, ViewKVs, DocIdKeys}} ->
- NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys),
+ {Go, {Seq, ViewKVs, DocIdKeys, Log}} ->
+ NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Log),
if Go == stop ->
Parent ! {new_state, NewState};
true ->
@@ -185,17 +185,17 @@ start_query_server(State) ->
accumulate_writes(State, W, Acc0) ->
- {Seq, ViewKVs, DocIdKVs} = case Acc0 of
- nil -> {0, [{V#mrview.id_num, []} || V <- State#mrst.views], []};
+ {Seq, ViewKVs, DocIdKVs, Log} = case Acc0 of
+ nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new()};
_ -> Acc0
end,
case couch_work_queue:dequeue(W) of
closed when Seq == 0 ->
stop;
closed ->
- {stop, {Seq, ViewKVs, DocIdKVs}};
+ {stop, {Seq, ViewKVs, DocIdKVs, Log}};
{ok, Info} ->
- {_, _, NewIds} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs),
+ {_, _, NewIds, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Log),
case accumulate_more(length(NewIds)) of
true -> accumulate_writes(State, W, Acc);
false -> {ok, Acc}
@@ -212,66 +212,100 @@ accumulate_more(NumDocIds) ->
andalso CurrMem < list_to_integer(MinSize).
-merge_results([], SeqAcc, ViewKVs, DocIdKeys) ->
- {SeqAcc, ViewKVs, DocIdKeys};
-merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) ->
- Fun = fun(RawResults, {VKV, DIK}) ->
- merge_results(RawResults, VKV, DIK)
+merge_results([], SeqAcc, ViewKVs, DocIdKeys, Log) ->
+ {SeqAcc, ViewKVs, DocIdKeys, Log};
+merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
+ Fun = fun(RawResults, {VKV, DIK, Log2}) ->
+ merge_results(RawResults, VKV, DIK, Log2)
end,
- {ViewKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs, DocIdKeys}, Results),
- merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1).
+ {ViewKVs1, DocIdKeys1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Log},
+ Results),
+ merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1,
+ Log1).
-merge_results({DocId, []}, ViewKVs, DocIdKeys) ->
- {ViewKVs, [{DocId, []} | DocIdKeys]};
-merge_results({DocId, RawResults}, ViewKVs, DocIdKeys) ->
+merge_results({DocId, _Seq, []}, ViewKVs, DocIdKeys, Log) ->
+ {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, [], Log)};
+merge_results({DocId, Seq, RawResults}, ViewKVs, DocIdKeys, Log) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
- {ViewKVs1, ViewIdKeys} = insert_results(DocId, Results, ViewKVs, [], []),
- {ViewKVs1, [ViewIdKeys | DocIdKeys]}.
+ {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Results, ViewKVs, [],
+ [], Log),
+ {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
-insert_results(DocId, [], [], ViewKVs, ViewIdKeys) ->
- {lists:reverse(ViewKVs), {DocId, ViewIdKeys}};
-insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) ->
+insert_results(DocId, _Seq, [], [], ViewKVs, ViewIdKeys, Log) ->
+ {lists:reverse(ViewKVs), {DocId, ViewIdKeys}, Log};
+insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
+ VIdKeys, Log) ->
CombineDupesFun = fun
- ({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys}) ->
- {[{Key, {dups, [Val | Vals]}} | Rest], IdKeys};
- ({Key, Val1}, {[{Key, Val2} | Rest], IdKeys}) ->
- {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys};
- ({Key, _}=KV, {Rest, IdKeys}) ->
- {[KV | Rest], [{Id, Key} | IdKeys]}
+ ({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys, Log2}) ->
+ {[{Key, {dups, [Val | Vals]}} | Rest], IdKeys, Log2};
+ ({Key, Val1}, {[{Key, Val2} | Rest], IdKeys, Log2}) ->
+ {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
+ ({Key, _}=KV, {Rest, IdKeys, Log2}) ->
+ {[KV | Rest], [{Id, Key} | IdKeys],
+ dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
end,
- InitAcc = {[], VIdKeys},
+ InitAcc = {[], VIdKeys, Log},
couch_stats:increment_counter([couchdb, mrview, emits], length(KVs)),
- {Duped, VIdKeys0} = lists:foldl(CombineDupesFun, InitAcc, lists:sort(KVs)),
+ {Duped, VIdKeys0, Log1} = lists:foldl(CombineDupesFun, InitAcc,
+ lists:sort(KVs)),
FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs,
- insert_results(DocId, RKVs, RVKVs, [{Id, FinalKVs} | VKVAcc], VIdKeys0).
+ FinalSKVs = [{{Seq, Key}, {DocId, Val}} || {Key, Val} <- Duped] ++ SKVs,
+ insert_results(DocId, Seq, RKVs, RVKVs,
+ [{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
-write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
+write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
#mrst{
id_btree=IdBtree,
+ log_btree=LogBtree,
first_build=FirstBuild
} = State,
{ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
ToRemByView = collapse_rem_keys(ToRemove, dict:new()),
- UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, KVs}) ->
+ {ok, SeqsToAdd, SeqsToRemove, LogBtree2} = case LogBtree of
+ nil -> {ok, undefined, undefined, nil};
+ _ -> update_log(LogBtree, Log, UpdateSeq, FirstBuild)
+ end,
+
+ UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
true -> UpdateSeq;
_ -> View#mrview.update_seq
end,
- View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}
+
+ %% store the view changes.
+ {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
+ true ->
+ SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+ SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+ SKVs1 = SKVs ++ SToAdd,
+ {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
+ SKVs1, SToRem),
+
+ {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
+ couch_mrview_util:to_key_seq(SKVs1),
+ couch_mrview_util:to_key_seq(SToRem)),
+ {SBt, KSbt};
+ _ -> {nil, nil}
+ end,
+ View#mrview{btree=VBtree2,
+ seq_btree=SeqBtree2,
+ key_byseq_btree=KeyBySeqBtree2,
+ update_seq=NewUpdateSeq}
end,
State#mrst{
views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs),
update_seq=UpdateSeq,
- id_btree=IdBtree2
+ id_btree=IdBtree2,
+ log_btree=LogBtree2
}.
@@ -284,6 +318,84 @@ update_id_btree(Btree, DocIdKeys, _) ->
ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
+walk_log(BTree, Fun, Acc, Ids) ->
+ WrapFun = fun(KV, _Offset, Acc2) ->
+ Fun(KV, Acc2)
+ end,
+ lists:foldl(fun(Id, Acc1) ->
+ Opt = [{start_key, Id}, {end_key, Id}],
+ {ok, _, A} = couch_btree:fold(BTree, WrapFun, Acc1, Opt),
+ A
+ end, Acc, Ids).
+
+update_log(Btree, Log, _UpdatedSeq, true) ->
+ ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
+ DIKeys /= []],
+ {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
+ {ok, dict:new(), dict:new(), LogBtree2};
+update_log(Btree, Log, UpdatedSeq, _) ->
+ %% build list of updated keys and Id
+ {ToLook, Updated} = dict:fold(fun
+ (Id, [], {IdsAcc, KeysAcc}) ->
+ {[Id | IdsAcc], KeysAcc};
+ (Id, DIKeys, {IdsAcc, KeysAcc}) ->
+ KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}},
+ KeysAcc2) ->
+ [{Id, ViewId, Key} | KeysAcc2]
+ end, KeysAcc, DIKeys),
+ {[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
+
+ io:format("updated ~p~n", [Updated]),
+ RemValue = {[{<<"_removed">>, true}]},
+ {Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
+ {Log2, AddAcc2, DelAcc2}) ->
+
+ {Log3, AddAcc3, DelAcc3} = lists:foldl(fun({ViewId,{Key, Seq, Op}},
+ {Log4, AddAcc4, DelAcc4}) ->
+
+ case lists:member({DocId, ViewId, Key}, Updated) of
+ true ->
+ %% the log is updated, deleted old
+ %% record from the view
+ DelAcc5 = dict:append(ViewId, {Seq, Key},
+ DelAcc4),
+ {Log4, AddAcc4, DelAcc5};
+ false when Op /= del ->
+ %% an update operation has been
+ %% logged for this key. We must now
+ %% record it as deleted in the
+ %% log, remove the old record in
+ %% the view and update the view
+ %% with a removed record.
+ Log5 = dict:append(DocId,
+ {ViewId,
+ {Key,UpdatedSeq, del}},
+ Log4),
+ DelAcc5 = dict:append(ViewId, {Seq, Key},
+ DelAcc4),
+ AddAcc5 = dict:append(ViewId,
+ {{UpdatedSeq, Key},
+ {DocId, RemValue}},
+ AddAcc4),
+ {Log5, AddAcc5, DelAcc5};
+ false ->
+ %% the key has already been
+ %% registered in the view as
+ %% deleted, make sure to add it
+ %% to the new log.
+ Log5 = dict:append(DocId,
+ {ViewId,
+ {Key, Seq, del}}, Log4),
+ {Log5, AddAcc4, DelAcc4}
+ end
+ end, {Log2, AddAcc2, DelAcc2}, VIdKeys),
+ {ok, {Log3, AddAcc3, DelAcc3}}
+ end, {Log, dict:new(), dict:new()}, ToLook),
+
+ ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
+ %% store the new logs
+ {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
+ {ok, AddAcc, DelAcc, LogBtree2}.
collapse_rem_keys([], Acc) ->
Acc;
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/44c07c30/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 7fc60f8..cd25887 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -21,13 +21,16 @@
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
--export([calculate_external_size/1]).
+-export([calculate_external_size/3]).
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
-export([extract_view/4, extract_view_reduce/1]).
-export([get_view_keys/1, get_view_queries/1]).
-export([set_view_type/3]).
+-export([changes_key_opts/2]).
+-export([fold_changes/4]).
+-export([to_key_seq/1]).
-define(MOD, couch_mrview_index).
@@ -91,15 +94,18 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
[Name, Else]),
DictBySrcAcc
end,
+ {DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}),
+ SeqIndexed = proplists:get_value(<<"seq_indexed">>, DesignOpts, false),
+
{RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
- NumViews = fun({_, View}, N) -> {View#mrview{id_num=N}, N+1} end,
+ NumViews = fun({_, View}, N) ->
+ {View#mrview{id_num=N, seq_indexed=SeqIndexed}, N+1}
+ end,
{Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>),
- {DesignOpts} = couch_util:get_value(<<"options">>, Fields, {[]}),
- {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
Lib = couch_util:get_value(<<"lib">>, RawViews, {[]}),
IdxState = #mrst{
@@ -108,7 +114,8 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
lib=Lib,
views=Views,
language=Language,
- design_opts=DesignOpts
+ design_opts=DesignOpts,
+ seq_indexed=SeqIndexed
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
@@ -152,7 +159,8 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
UpdateSeq = couch_db:get_update_seq(Db),
PurgeSeq = couch_db:get_purge_seq(Db),
- Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq}),
+ Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
+ State#mrst.seq_indexed}),
couch_index_util:hexsig(couch_util:md5(Bin));
view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
view_sig(Db, State, View, Args);
@@ -160,11 +168,12 @@ view_sig(_Db, State, View, Args0) ->
Sig = State#mrst.sig,
UpdateSeq = View#mrview.update_seq,
PurgeSeq = View#mrview.purge_seq,
+ SeqIndexed = View#mrview.seq_indexed,
Args = Args0#mrargs{
preflight_fun=undefined,
extra=[]
},
- Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, Args}),
+ Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, SeqIndexed, Args}),
couch_index_util:hexsig(couch_util:md5(Bin)).
@@ -173,7 +182,8 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
seq=0,
purge_seq=couch_db:get_purge_seq(Db),
id_btree_state=nil,
- view_states=[{nil, 0, 0} || _ <- Views]
+ log_btree_state=nil,
+ view_states=[{nil, nil, nil, 0, 0} || _ <- Views]
},
init_state(Db, Fd, State, Header);
% read <= 1.2.x header record and transpile it to >=1.3.x
@@ -187,25 +197,31 @@ init_state(Db, Fd, State, #index_header{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=IdBtreeState,
- view_states=ViewStates
+ log_btree_state=nil,
+ view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates]
});
init_state(Db, Fd, State, Header) ->
- #mrst{language=Lang, views=Views} = State,
+ #mrst{language=Lang, views=Views, seq_indexed=SeqIndexed} = State,
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=IdBtreeState,
+ log_btree_state=LogBtreeState,
view_states=ViewStates
} = Header,
StateUpdate = fun
- ({_, _, _}=St) -> St;
- (St) -> {St, 0, 0}
+ ({_, _, _, _, _}=St) -> St;
+ (St) -> {St, nil, nil, 0, 0}
end,
ViewStates2 = lists:map(StateUpdate, ViewStates),
IdBtOpts = [{compression, couch_db:compression(Db)}],
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
+ {ok, LogBtree} = case SeqIndexed of
+ true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
+ false -> {ok, nil}
+ end,
OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views),
@@ -216,11 +232,20 @@ init_state(Db, Fd, State, Header) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
+ log_btree=LogBtree,
views=Views2
}.
+less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
+ case couch_ejson_compare:less(SeqA, SeqB) of
+ 0 ->
+ couch_ejson_compare:less_json(JsonA, JsonB);
+ Result ->
+ Result < 0
+ end.
+
-open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
+open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
ReduceFun =
fun(reduce, KVs) ->
@@ -245,7 +270,23 @@ open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
{compression, couch_db:compression(Db)}
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- View#mrview{btree=Btree, update_seq=USeq, purge_seq=PSeq}.
+
+ {SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
+ true ->
+ ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+ {compression, couch_db:compression(Db)}],
+ {ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
+ {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, ViewBtOpts),
+ {SBt, KSBt};
+ false ->
+ {nil, nil}
+ end,
+
+ View#mrview{btree=Btree,
+ seq_btree=SeqBtree,
+ key_byseq_btree=KeyBySeqBtree,
+ update_seq=USeq,
+ purge_seq=PSeq}.
temp_view_to_ddoc({Props}) ->
@@ -300,7 +341,6 @@ fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
end,
{ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
-
fold_fun(_Fun, [], _, Acc) ->
{ok, Acc};
fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
@@ -311,6 +351,12 @@ fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
{stop, Acc2}
end.
+fold_changes(Bt, Fun, Acc, Opts) ->
+ WrapperFun = fun(KV, _Reds, Acc2) ->
+ Fun(changes_expand_dups([KV], []), Acc2)
+ end,
+ {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
+
fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
#mrview{
@@ -495,21 +541,34 @@ make_header(State) ->
update_seq=Seq,
purge_seq=PurgeSeq,
id_btree=IdBtree,
+ log_btree=LogBtree,
views=Views
} = State,
- ViewStates = [
- {
- couch_btree:get_state(V#mrview.btree),
- V#mrview.update_seq,
- V#mrview.purge_seq
- }
- ||
- V <- Views
- ],
+
+ ViewStates = lists:foldr(fun(V, Acc) ->
+ {SeqBtState, KSeqBtState} = case V#mrview.seq_indexed of
+ true ->
+ {couch_btree:get_state(V#mrview.seq_btree),
+ couch_btree:get_state(V#mrview.key_byseq_btree)};
+ _ -> {nil, nil}
+ end,
+ [{couch_btree:get_state(V#mrview.btree),
+ SeqBtState,
+ KSeqBtState,
+ V#mrview.update_seq,
+ V#mrview.purge_seq} | Acc]
+ end, [], Views),
+
+ LogBtreeState = case LogBtree of
+ nil -> nil;
+ _ -> couch_btree:get_state(LogBtree)
+ end,
+
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
id_btree_state=couch_btree:get_state(IdBtree),
+ log_btree_state= LogBtreeState,
view_states=ViewStates
}.
@@ -567,7 +626,9 @@ reset_state(State) ->
qserver=nil,
update_seq=0,
id_btree=nil,
- views=[View#mrview{btree=nil} || View <- State#mrst.views]
+ log_btree=nil,
+ views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+ || View <- State#mrst.views]
}.
@@ -636,11 +697,62 @@ reverse_key_default(<<255>>) -> <<>>;
reverse_key_default(Key) -> Key.
-calculate_external_size(Views) ->
- SumFun = fun(#mrview{btree=Bt}, Acc) ->
- sum_btree_sizes(Acc, couch_btree:size(Bt))
+changes_key_opts(StartSeq, Args) ->
+ changes_key_opts(StartSeq, Args, []).
+
+
+changes_key_opts(StartSeq, #mrargs{keys=undefined, direction=Dir}=Args, Extra) ->
+ [[{dir, Dir}] ++ changes_skey_opts(StartSeq, Args) ++
+ changes_ekey_opts(StartSeq, Args) ++ Extra];
+changes_key_opts(StartSeq, #mrargs{keys=Keys, direction=Dir}=Args, Extra) ->
+ lists:map(fun(K) ->
+ [{dir, Dir}]
+ ++ changes_skey_opts(StartSeq, Args#mrargs{start_key=K})
+ ++ changes_ekey_opts(StartSeq, Args#mrargs{end_key=K})
+ ++ Extra
+ end, Keys).
+
+
+changes_skey_opts(StartSeq, #mrargs{start_key=undefined}) ->
+ [{start_key, [<<>>, StartSeq+1]}];
+changes_skey_opts(StartSeq, #mrargs{start_key=SKey,
+ start_key_docid=SKeyDocId}) ->
+ [{start_key, {[SKey, StartSeq+1], SKeyDocId}}].
+
+
+changes_ekey_opts(_StartSeq, #mrargs{end_key=undefined}) ->
+ [];
+changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
+ end_key_docid=EKeyDocId,
+ direction=Dir}=Args) ->
+ EndSeq = case Dir of
+ fwd -> 16#10000000;
+ rev -> 0
+ end,
+
+ case Args#mrargs.inclusive_end of
+ true -> [{end_key, {[EKey, EndSeq], EKeyDocId}}];
+ false -> [{end_key_gt, {[EKey, EndSeq], EKeyDocId}}]
+ end.
+
+
+
+calculate_external_size(IdBt, LogBt, Views) ->
+ SumFun = fun
+ (#mrview{btree=Bt, seq_btree=nil}, Acc) ->
+ sum_btree_sizes(Acc, couch_btree:size(Bt));
+ (#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
+ Acc1 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+ Acc2 = sum_btree_sizes(Acc1, couch_btree:size(SBt)),
+ sum_btree_sizes(Acc2, couch_btree:size(KSBt))
+ end,
+ Size = case LogBt of
+ nil ->
+ lists:foldl(SumFun, couch_btree:size(IdBt), Views);
+ _ ->
+ lists:foldl(SumFun, couch_btree:size(IdBt) +
+ couch_btree:size(LogBt), Views)
end,
- Size = lists:foldl(SumFun, 0, Views),
{ok, Size}.
@@ -669,6 +781,19 @@ expand_dups([KV | Rest], Acc) ->
expand_dups(Rest, [KV | Acc]).
+changes_expand_dups([], Acc) ->
+ lists:reverse(Acc);
+changes_expand_dups([{{[Key, Seq], DocId}, {dups, Vals}} | Rest], Acc) ->
+ Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+ changes_expand_dups(Rest, Expanded ++ Acc);
+changes_expand_dups([{{Key, Seq}, {DocId, {dups, Vals}}} | Rest], Acc) ->
+ Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+ changes_expand_dups(Rest, Expanded ++ Acc);
+changes_expand_dups([{{[Key, Seq], DocId}, Val} | Rest], Acc) ->
+ changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]);
+changes_expand_dups([{{Key, Seq}, {DocId, Val}} | Rest], Acc) ->
+ changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]).
+
maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
[];
maybe_load_doc(Db, #doc_info{}=DI, #mrargs{conflicts=true, doc_options=Opts}) ->
@@ -718,6 +843,9 @@ mrverror(Mesg) ->
throw({query_parse_error, Mesg}).
+to_key_seq(L) ->
+ [{{[Key, Seq], DocId}, Val} || {{Seq, Key}, {DocId, Val}} <- L].
+
%% Updates 1.2.x or earlier view files to 1.3.x or later view files
%% transparently, the first time the 1.2.x view file is opened by
%% 1.3.x or later.
[17/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
fix case clause
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/42ff2003
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/42ff2003
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/42ff2003
Branch: refs/heads/master
Commit: 42ff200319232f3ce90ead2ff6d82c09cf02406e
Parents: a162355
Author: benoitc <bc...@gmail.com>
Authored: Thu Feb 27 21:55:46 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_changes.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/42ff2003/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index c31624e..735ded8 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -144,7 +144,7 @@ changes_timeout(Options) ->
view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
view_options=Options, since=Since,
callback=Callback, acc=UserAcc}=State) ->
- Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {Go, Acc2, OldSeq}) ->
+ Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {_Go, Acc2, OldSeq}) ->
LastSeq = if OldSeq < Seq -> Seq;
true -> OldSeq
end,
[28/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Fix removal of keys from view seq index
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/df3d28d0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/df3d28d0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/df3d28d0
Branch: refs/heads/master
Commit: df3d28d0d25c6d7a4d053ed13c493e78d23735ba
Parents: c0ad453
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Aug 28 21:22:55 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/df3d28d0/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 66db969..2dcce06 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -265,13 +265,17 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
- {ViewKVs, [{DocId, Rev, []} | DocIdKeys], dict:store(DocId, [], Log)};
+ {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
- {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [],
- [], Log),
- {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
+ case Results of
+ [[],[]] ->
+ {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
+ _ ->
+ {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),
+ {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}
+ end.
insert_results(DocId, _Seq, _Rev, [], [], ViewKVs, ViewIdKeys, Log) ->
@@ -285,7 +289,7 @@ insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVA
{[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
({Key, Value}, {Rest, IdKeys, Log2}) ->
{[{Key, Value} | Rest], [{Id, Key} | IdKeys],
- dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
+ dict:append({DocId, Rev}, {Id, {Key, Seq, add}}, Log2)}
end,
InitAcc = {[], VIdKeys, Log},
couch_stats:increment_counter([couchdb, mrview, emits], length(KVs)),
@@ -375,12 +379,16 @@ update_log(Btree, Log, _UpdatedSeq, true) ->
{ok, dict:new(), dict:new(), LogBtree2};
update_log(Btree, Log, UpdatedSeq, _) ->
%% build list of updated keys and Id
+ Revs = dict:from_list(dict:fetch_keys(Log)),
+ Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
+ dict:store(Id, DIKeys, Acc)
+ end, dict:new(), Log),
{ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
[{Id, ViewId, Key} | KeysAcc2]
end, KeysAcc, DIKeys),
{[Id | IdsAcc], KeysAcc1}
- end, {[], []}, Log),
+ end, {[], []}, Log0),
MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
@@ -399,7 +407,8 @@ update_log(Btree, Log, UpdatedSeq, _) ->
LogValue = {ViewId, {Key, UpdatedSeq, del}},
Log5 = dict:append(DocId, LogValue, Log4),
DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
- AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL}},
+ Rev = dict:fetch(DocId, Revs),
+ AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
{Log5, AddAcc5, DelAcc5};
false ->
@@ -409,7 +418,7 @@ update_log(Btree, Log, UpdatedSeq, _) ->
{Log5, AddAcc4, DelAcc4}
end
end, Acc, VIdKeys)
- end, {Log, dict:new(), dict:new()}, KVsToLook),
+ end, {Log0, dict:new(), dict:new()}, KVsToLook),
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
% store the new logs
[07/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: add API to retrieve view info.
Add couch_mrview:get_view_info/3 to retrieve the internal informations
of a view like the last update seq in this view or the number of rows.
Conflicts:
src/couch_mrview.erl
test/04-index-info.t
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/a043479b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/a043479b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/a043479b
Branch: refs/heads/master
Commit: a043479bc4182fa84190e7a2909bc1e344bd7236
Parents: 7bba23f
Author: benoitc <be...@apache.org>
Authored: Sat Feb 8 22:49:42 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 25 ++++++++++++++++++++++++-
1 file changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a043479b/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 8353961..5b5ba25 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -19,6 +19,7 @@
-export([count_view_changes_since/4, count_view_changes_since/5]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
+-export([get_view_info/3]).
-export([refresh/2]).
-export([compact/2, compact/3, cancel_compaction/2]).
-export([cleanup/1]).
@@ -198,11 +199,33 @@ get_info(Db, DDoc) ->
trigger_update(Db, DDoc) ->
trigger_update(Db, DDoc, couch_db:get_update_seq(Db)).
-
trigger_update(Db, DDoc, UpdateSeq) ->
{ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
couch_index:trigger_update(Pid, UpdateSeq).
+%% get informations on a view
+get_view_info(Db, DDoc, VName) ->
+ {ok, {_, View}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+ #mrargs{}),
+
+ %% get the total number of rows
+ {ok, TotalRows} = couch_mrview_util:get_row_count(View),
+
+ %% get the total number of sequence logged in this view
+ SeqBtree = View#mrview.seq_btree,
+ {ok, TotalSeqs} = case SeqBtree of
+ nil -> {ok, 0};
+ _ ->
+ {ok, {Count, _Reds}} = couch_btree:full_reduce(SeqBtree),
+ {ok, Count}
+ end,
+
+ {ok, [{update_seq, View#mrview.update_seq},
+ {purge_seq, View#mrview.purge_seq},
+ {total_rows, TotalRows},
+ {total_seqs, TotalSeqs}]}.
+
+
%% @doc refresh a view index
refresh(#db{name=DbName}, DDoc) ->
refresh(DbName, DDoc);
[08/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
add supports of view changes in the _changes API
Now when the option `seq_indexed=true` is set in the design doc, the
view filter in _changes will use it to retrieve the results. Compared to
the current way, using a view index will be faster to retrieve changes.
It also gives the possibility to filter changes by key or get changes in
a key range. All the view options can be used.
Note 1: if someone is trying to filter a changes with view options when
the views are not indexed by sequence, a 400 error will be returned.
Note 2: The changes will only be returned when the view is updated if
seq_indexed=true
Conflicts:
src/couch_mrview_http.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/dfe991f0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/dfe991f0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/dfe991f0
Branch: refs/heads/master
Commit: dfe991f0fcfe15740c0927c6be3cd9504114b894
Parents: 1c24c42
Author: benoitc <bc...@gmail.com>
Authored: Fri Feb 7 15:38:34 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_changes.erl | 6 ++----
src/couch_mrview_http.erl | 8 ++++++--
src/couch_mrview_index.erl | 20 +++++++++++++++++---
test/09-index-events.t | 21 +++++++++++++++++----
4 files changed, 42 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/dfe991f0/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index 2b8f910..a0e5281 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -141,10 +141,8 @@ view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
true -> OldSeq
end,
- case Callback(KV, Acc2) of
- {ok, Acc3} -> {ok, {Go, Acc3, LastSeq}};
- {stop, Acc3} -> {stop, {stop, Acc3, LastSeq}}
- end
+ {Go, Acc3} = Callback(KV, Acc2),
+ {Go, {Go, Acc3, LastSeq}}
end,
Acc0 = {ok, UserAcc, Since},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/dfe991f0/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index aeaca8e..f3d3286 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -34,6 +34,10 @@
check_view_etag/3
]).
+-export([parse_boolean/1,
+ parse_int/1,
+ parse_pos_int/1]).
+
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -438,8 +442,10 @@ parse_boolean(true) ->
true;
parse_boolean(false) ->
false;
+
parse_boolean(Val) when is_binary(Val) ->
parse_boolean(?b2l(Val));
+
parse_boolean(Val) ->
case string:to_lower(Val) of
"true" -> true;
@@ -449,7 +455,6 @@ parse_boolean(Val) ->
throw({query_parse_error, ?l2b(Msg)})
end.
-
parse_int(Val) when is_integer(Val) ->
Val;
parse_int(Val) ->
@@ -461,7 +466,6 @@ parse_int(Val) ->
throw({query_parse_error, ?l2b(Msg)})
end.
-
parse_pos_int(Val) ->
case parse_int(Val) of
IntVal when IntVal >= 0 ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/dfe991f0/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 0b9b236..7598f13 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -39,8 +39,10 @@ get(Property, State) ->
Opts = State#mrst.design_opts,
IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
+ SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
if IncDesign -> [include_design]; true -> [] end
- ++ if LocalSeq -> [local_seq]; true -> [] end;
+ ++ if LocalSeq -> [local_seq]; true -> [] end
+ ++ if SeqIndexed -> [seq_indexed]; true -> [] end;
fd ->
State#mrst.fd;
language ->
@@ -56,13 +58,24 @@ get(Property, State) ->
language = Lang,
update_seq = UpdateSeq,
purge_seq = PurgeSeq,
- views = Views
+ views = Views,
+ design_opts = Opts
} = State,
{ok, FileSize} = couch_file:bytes(Fd),
{ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
LogBtree,
Views),
ActiveSize = ExternalSize + couch_btree:size(Btree),
+
+ IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
+ LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
+ SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+ UpdateOptions =
+ if IncDesign -> [<<"include_design">>]; true -> [] end
+ ++ if LocalSeq -> [<<"local_seq">>]; true -> [] end
+ ++ if SeqIndexed -> [<<"seq_indexed">>]; true -> [] end,
+
+
{ok, [
{signature, list_to_binary(couch_index_util:hexsig(Sig))},
{language, Lang},
@@ -74,7 +87,8 @@ get(Property, State) ->
{external, ExternalSize}
]}},
{update_seq, UpdateSeq},
- {purge_seq, PurgeSeq}
+ {purge_seq, PurgeSeq},
+ {update_options, UpdateOptions}
]};
Other ->
throw({unknown_index_property, Other})
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/dfe991f0/test/09-index-events.t
----------------------------------------------------------------------
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 1489e4e..6cc1e9c 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -15,7 +15,7 @@
% the License.
main(_) ->
- etap:plan(4),
+ etap:plan(5),
case (catch test()) of
ok ->
etap:end_tests();
@@ -29,11 +29,18 @@ main(_) ->
test() ->
test_util:start_couch(),
{ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+ test_info(Db),
test_update_event(Db),
test_delete_event(Db),
test_util:stop_couch(),
ok.
+test_info(Db) ->
+ {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
+ etap:is(getval(update_options, Info), [<<"seq_indexed">>],
+ "update options OK"),
+ ok.
+
test_update_event(Db) ->
{ok, Pid} = couch_index_event:start_link(self()),
etap:ok(is_pid(Pid), "event handler added"),
@@ -47,11 +54,13 @@ test_update_event(Db) ->
couch_index_event:stop(Pid).
test_delete_event(Db) ->
- ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+ ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+ timer:sleep(300),
{ok, Pid} = couch_index_event:start_link(self()),
+ etap:ok(is_pid(Pid), "delete event handler added"),
- etap:ok(is_pid(Pid), "event handler added"),
- couch_mrview_test_util:delete_db(<<"foo">>),
+
+ catch couch_mrview_test_util:delete_db(<<"foo">>),
Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
couch_mrview_index}},
receive
@@ -59,3 +68,7 @@ test_delete_event(Db) ->
etap:is(Event, Expect, "index delete events OK")
end,
couch_index_event:stop(Pid).
+
+getval(Key, PL) ->
+ {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
+ Val.
[14/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Throw 400 on _view_changes when by-seq indexing is disabled
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/cc45433e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/cc45433e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/cc45433e
Branch: refs/heads/master
Commit: cc45433e9630b69d0e37fb3c5a893b7d1e1fa4e9
Parents: c8394f9
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Aug 22 20:28:44 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_http.erl | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/cc45433e/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 75c3cb5..2b396b0 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -62,7 +62,20 @@ handle_reindex_req(Req, _Db, _DDoc) ->
couch_httpd:send_method_not_allowed(Req, "POST").
-handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
+handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, DDoc) ->
+ {DDocBody} = DDoc#doc.body,
+ case lists:keyfind(<<"options">>, 1, DDocBody) of
+ {<<"options">>, {Options}} when is_list(Options) ->
+ case lists:keyfind(<<"seq_indexed">>, 1, Options) of
+ {<<"seq_indexed">>, true} ->
+ ok;
+ _ ->
+ throw({bad_request, "view changes not enabled"})
+ end;
+ _ ->
+ throw({bad_request, "view changes not enabled"})
+ end,
+
ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
[12/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
add index update events notifications
Conflicts:
src/couch_mrview_updater.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/18b5f6ff
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/18b5f6ff
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/18b5f6ff
Branch: refs/heads/master
Commit: 18b5f6ff54683fca633cd1678522936a03598546
Parents: 7a65916
Author: benoitc <be...@apache.org>
Authored: Thu Jan 30 09:21:06 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 3 +++
test/09-index-events.t | 46 +++++++++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/18b5f6ff/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 16645aa..7b24e42 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -194,6 +194,9 @@ write_results(Parent, State) ->
Parent ! {new_state, NewState};
true ->
send_partial(NewState#mrst.partial_resp_pid, NewState),
+ % notifify the view update
+ couch_index_event:notify({index_update, {DbName, IdxName,
+ couch_mrview_index}}),
write_results(Parent, NewState)
end
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/18b5f6ff/test/09-index-events.t
----------------------------------------------------------------------
diff --git a/test/09-index-events.t b/test/09-index-events.t
new file mode 100644
index 0000000..90654b8
--- /dev/null
+++ b/test/09-index-events.t
@@ -0,0 +1,46 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+ etap:plan(2),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ timer:sleep(300),
+ ok.
+
+test() ->
+ test_util:start_couch(),
+ {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+ test_update_event(Db),
+ test_util:stop_couch(),
+ ok.
+
+test_update_event(Db) ->
+ {ok, Pid} = couch_index_event:start_link(self()),
+ etap:ok(is_pid(Pid), "event handler added"),
+ ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+ Expect = {index_update, {<<"foo">>, <<"_design/bar">>,
+ couch_mrview_index}},
+ receive
+ Event ->
+ etap:is(Event, Expect, "index update events OK")
+ end,
+ couch_index_event:stop(Pid).
[39/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
fix removal of duplicate keys from seq tree
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/11d4c17e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/11d4c17e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/11d4c17e
Branch: refs/heads/master
Commit: 11d4c17e4ca729832396d8b393b8c0b95bad7bc0
Parents: 2e0be35
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Sep 16 12:52:54 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:42 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/11d4c17e/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index b7f91e1..c98c2f5 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -273,8 +273,8 @@ merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
- case Results of
- [[],[]] ->
+ case lists:flatten(Results) of
+ [] ->
{ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
_ ->
{ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),
[13/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: add couch_mrview:refresh/2 function
function to refresh a view index.
Conflicts:
src/couch_mrview.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/7a65916b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/7a65916b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/7a65916b
Branch: refs/heads/master
Commit: 7a65916ba27c77d54191d9ac35009ad387aa0bf6
Parents: 9656222
Author: benoitc <be...@apache.org>
Authored: Tue Jan 28 23:23:37 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 19 +++++++++++++++++++
test/08-changes_since.t | 4 ----
2 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/7a65916b/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 2f68fc0..8353961 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -19,6 +19,7 @@
-export([count_view_changes_since/4, count_view_changes_since/5]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
+-export([refresh/2]).
-export([compact/2, compact/3, cancel_compaction/2]).
-export([cleanup/1]).
@@ -202,6 +203,24 @@ trigger_update(Db, DDoc, UpdateSeq) ->
{ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
couch_index:trigger_update(Pid, UpdateSeq).
+%% @doc refresh a view index
+refresh(#db{name=DbName}, DDoc) ->
+ refresh(DbName, DDoc);
+
+refresh(Db, DDoc) ->
+ UpdateSeq = couch_util:with_db(Db, fun(WDb) ->
+ couch_db:get_update_seq(WDb)
+ end),
+
+ case couch_index_server:get_index(couch_mrview_index, Db, DDoc) of
+ {ok, Pid} ->
+ case catch couch_index:get_state(Pid, UpdateSeq) of
+ {ok, _} -> ok;
+ Error -> {error, Error}
+ end;
+ Error ->
+ {error, Error}
+ end.
compact(Db, DDoc) ->
compact(Db, DDoc, []).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/7a65916b/test/08-changes_since.t
----------------------------------------------------------------------
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 5a10f4c..58c92e9 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -146,10 +146,6 @@ test_remove_key(Db) ->
]},
etap:is(Result3, Expect2, "removed key OK.").
-
-
-
-
run_query(Db, Since, Opts) ->
Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
{ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
[04/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: fix 10-index-changes.t
proper test to check the indexer.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/7bba23ff
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/7bba23ff
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/7bba23ff
Branch: refs/heads/master
Commit: 7bba23ffced699b8637116ad7f8e473b6b0b2d13
Parents: 5691328
Author: benoitc <be...@apache.org>
Authored: Sat Feb 8 20:20:47 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
test/10-index-changes.t | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/7bba23ff/test/10-index-changes.t
----------------------------------------------------------------------
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index f53e9ed..ce79eac 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -175,13 +175,13 @@ test_stream(Db) ->
test_indexer(Db) ->
- Result = run_query(Db, [{since, 14}]),
+ Result = run_query(Db, [{since, 14}, refresh]),
Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
etap:is(Result, Expect, "refresh index by hand OK."),
{ok, Db1} = save_doc(Db, 15),
- timer:sleep(1000),
- Result1 = run_query(Db, [{since, 14}]),
+ timer:sleep(1500),
+ Result1 = run_query(Db1, [{since, 14}], false),
Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
{{16,15,<<"15">>},15}]},
etap:is(Result1, Expect1, "changes indexed in background OK."),
@@ -195,6 +195,9 @@ save_doc(Db, Id) ->
couch_db:reopen(Db).
run_query(Db, Opts) ->
+ run_query(Db, Opts, true).
+
+run_query(Db, Opts, Refresh) ->
Fun = fun
(stop, {LastSeq, Acc}) ->
{ok, LastSeq, Acc};
@@ -203,7 +206,12 @@ run_query(Db, Opts) ->
(Event, Acc) ->
{ok, [Event | Acc]}
end,
- couch_mrview:refresh(Db, <<"_design/bar">>),
+ case Refresh of
+ true ->
+ couch_mrview:refresh(Db, <<"_design/bar">>);
+ false ->
+ ok
+ end,
{ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
<<"baz">>, Fun, [], Opts),
{ok, LastSeq, lists:reverse(R)}.
[31/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Port view changes tests to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/d6771df7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/d6771df7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/d6771df7
Branch: refs/heads/master
Commit: d6771df75c68d0796d4758e02b653a2d211707e3
Parents: 487df67
Author: Benjamin Bastian <be...@gmail.com>
Authored: Mon Sep 8 15:10:17 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_test_util.erl | 2 +-
test/08-changes_since.t | 156 ------------------
test/09-index-events.t | 73 ---------
test/10-index-changes.t | 216 ------------------------
test/couch_mrview_changes_since_tests.erl | 166 +++++++++++++++++++
test/couch_mrview_index_changes_tests.erl | 217 +++++++++++++++++++++++++
6 files changed, 384 insertions(+), 446 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/src/couch_mrview_test_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index 1d3d788..c8da1ee 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -34,7 +34,7 @@ new_db(Name, Type) ->
save_docs(Db, [ddoc(Type)]).
delete_db(Name) ->
- couch_server:delete(Name, [{user_ctx, ?ADMIN}]).
+ couch_server:delete(Name, [?ADMIN_USER]).
save_docs(Db, Docs) ->
{ok, _} = couch_db:update_docs(Db, Docs, []),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/test/08-changes_since.t
----------------------------------------------------------------------
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
deleted file mode 100644
index 3107c1a..0000000
--- a/test/08-changes_since.t
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-main(_) ->
- etap:plan(14),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- timer:sleep(300),
- ok.
-
-test() ->
- test_util:start_couch(),
-
- {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
-
- test_basic(Db),
- test_range(Db),
- test_basic_since(Db),
- test_range_since(Db),
- test_basic_count(Db),
- test_range_count(Db),
- test_basic_count_since(Db),
- test_range_count_since(Db),
- test_compact(Db),
- test_remove_key(Db),
- catch test_util:stop_couch(),
- ok.
-
-
-test_basic(Db) ->
- Result = run_query(Db, 0, []),
- Expect = {ok, [
- {{2, 1, <<"1">>}, 1},
- {{3, 10, <<"10">>}, 10},
- {{4, 2, <<"2">>}, 2},
- {{5, 3, <<"3">>}, 3},
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5},
- {{8, 6, <<"6">>}, 6},
- {{9, 7, <<"7">>}, 7},
- {{10, 8, <<"8">>}, 8},
- {{11, 9, <<"9">>}, 9}
- ]},
- etap:is(Result, Expect, "Simple view query worked.").
-
-
-test_range(Db) ->
- Result = run_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
- Expect = {ok, [
- {{5, 3, <<"3">>}, 3},
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5}
- ]},
- etap:is(Result, Expect, "Query with range works.").
-
-test_basic_since(Db) ->
- Result = run_query(Db, 5, []),
- Expect = {ok, [
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5},
- {{8, 6, <<"6">>}, 6},
- {{9, 7, <<"7">>}, 7},
- {{10, 8, <<"8">>}, 8},
- {{11, 9, <<"9">>}, 9}
- ]},
- etap:is(Result, Expect, "Simple view query since 5 worked.").
-
-test_range_since(Db) ->
- Result = run_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
- Expect = {ok, [
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5}
- ]},
- etap:is(Result, Expect, "Query with range since 5 works.").
-
-test_basic_count(Db) ->
- Result = run_count_query(Db, 0, []),
- etap:is(Result, 10, "Simple view count worked.").
-
-test_range_count(Db) ->
- Result = run_count_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
- etap:is(Result, 3, "Count with range works.").
-
-test_basic_count_since(Db) ->
- Result = run_count_query(Db, 5, []),
- etap:is(Result, 6, "Simple view count since 5 worked.").
-
-test_range_count_since(Db) ->
- Result = run_count_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
- etap:is(Result, 2, "Count with range since 5 works.").
-
-test_compact(Db) ->
- Result = couch_mrview:compact(Db, <<"_design/bar">>),
- etap:is(Result, ok, "compact view is OK"),
- Count = run_count_query(Db, 0, []),
- etap:is(Count, 10, "compact view worked.").
-
-test_remove_key(Db) ->
- %% add new doc
- Doc = couch_mrview_test_util:doc(11),
- {ok, Rev} = couch_db:update_doc(Db, Doc, []),
- RevStr = couch_doc:rev_to_str(Rev),
- {ok, _} = couch_db:ensure_full_commit(Db),
- {ok, Db1} = couch_db:reopen(Db),
- Result = run_count_query(Db1, 0, []),
- etap:is(Result, 11, "Add new doc worked."),
- %% check new view key
- Result1 = run_query(Db1, 0, [{start_key, 11}, {end_key, 11}]),
- Expect = {ok, [
- {{12, 11, <<"11">>}, 11}
- ]},
- etap:is(Result1, Expect, "added key OK."),
-
- %% delete doc
- Doc2 = couch_doc:from_json_obj({[
- {<<"_id">>, <<"11">>},
- {<<"_rev">>, RevStr},
- {<<"_deleted">>, true}
- ]}),
- {ok, _} = couch_db:update_doc(Db1, Doc2, []),
- {ok, Db2} = couch_db:reopen(Db1),
- Result2 = run_count_query(Db2, 0, []),
- etap:is(Result2, 11, "removed key saved."),
- %% check new view key
- Result3 = run_query(Db2, 0, [{start_key, 11}, {end_key, 11}]),
- Expect2 = {ok, [
- {{13, 11, <<"11">>}, {[{<<"_removed">>, true}]}}
- ]},
- etap:is(Result3, Expect2, "removed key OK.").
-
-run_query(Db, Since, Opts) ->
- Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
- {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
- Since, Fun, Opts, []),
- {ok, lists:reverse(R)}.
-
-run_count_query(Db, Since, Opts) ->
- couch_mrview:count_view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
- Since, Opts).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/test/09-index-events.t
----------------------------------------------------------------------
diff --git a/test/09-index-events.t b/test/09-index-events.t
deleted file mode 100644
index 26d904d..0000000
--- a/test/09-index-events.t
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-main(_) ->
- etap:plan(5),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- timer:sleep(300),
- ok.
-
-test() ->
- test_util:start_couch(),
- {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
- test_info(Db),
- test_update_event(Db),
- test_delete_event(Db),
- test_util:stop_couch(),
- ok.
-
-test_info(Db) ->
- {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
- etap:is(getval(update_options, Info), [<<"seq_indexed">>],
- "update options OK"),
- ok.
-
-test_update_event(Db) ->
- {ok, Pid} = couch_index_event:start_link(self()),
- etap:ok(is_pid(Pid), "event handler added"),
- ok = couch_mrview:refresh(Db, <<"_design/bar">>),
- Expect = {index_update, {<<"foo">>, <<"_design/bar">>,
- couch_mrview_index}},
- receive
- Event ->
- etap:is(Event, Expect, "index update events OK")
- end,
- couch_index_event:stop(Pid).
-
-test_delete_event(Db) ->
- ok = couch_mrview:refresh(Db, <<"_design/bar">>),
- timer:sleep(300),
- {ok, Pid} = couch_index_event:start_link(self()),
- etap:ok(is_pid(Pid), "delete event handler added"),
-
-
- catch couch_mrview_test_util:delete_db(<<"foo">>),
- Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
- couch_mrview_index}},
- receive
- Event ->
- etap:is(Event, Expect, "index delete events OK")
- end,
- couch_index_event:stop(Pid).
-
-getval(Key, PL) ->
- {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
- Val.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/test/10-index-changes.t
----------------------------------------------------------------------
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
deleted file mode 100644
index 0c5e924..0000000
--- a/test/10-index-changes.t
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-main(_) ->
- etap:plan(8),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- timer:sleep(300),
- ok.
-
-test() ->
- test_util:start_couch(),
- {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
- test_normal_changes(Db),
- test_stream_once(Db),
- test_stream_once_since(Db),
- test_stream_once_timeout(Db),
- test_stream_once_heartbeat(Db),
- test_stream(Db),
- test_indexer(Db),
- test_util:stop_couch(),
- ok.
-
-test_normal_changes(Db) ->
- Result = run_query(Db, []),
- Expect = {ok, 11, [
- {{2, 1, <<"1">>}, 1},
- {{3, 10, <<"10">>}, 10},
- {{4, 2, <<"2">>}, 2},
- {{5, 3, <<"3">>}, 3},
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5},
- {{8, 6, <<"6">>}, 6},
- {{9, 7, <<"7">>}, 7},
- {{10, 8, <<"8">>}, 8},
- {{11, 9, <<"9">>}, 9}
- ]},
- etap:is(Result, Expect, "normal changes worked.").
-
-test_stream_once(Db) ->
- Result = run_query(Db, [{stream, once}]),
- Expect = {ok, 11, [
- {{2, 1, <<"1">>}, 1},
- {{3, 10, <<"10">>}, 10},
- {{4, 2, <<"2">>}, 2},
- {{5, 3, <<"3">>}, 3},
- {{6, 4, <<"4">>}, 4},
- {{7, 5, <<"5">>}, 5},
- {{8, 6, <<"6">>}, 6},
- {{9, 7, <<"7">>}, 7},
- {{10, 8, <<"8">>}, 8},
- {{11, 9, <<"9">>}, 9}
- ]},
- etap:is(Result, Expect, "stream once since 0 worked.").
-
-
-test_stream_once_since(Db) ->
- Self = self(),
- spawn(fun() ->
- Result = run_query(Db, [{since, 11},
- {stream, once}]),
- Self ! {result, Result}
- end),
-
- spawn(fun() ->
- timer:sleep(1000),
- {ok, Db1} = save_doc(Db, 11),
- couch_mrview:refresh(Db1, <<"_design/bar">>)
- end),
-
- Expect = {ok,12,[{{12,11,<<"11">>},11}]},
-
- receive
- {result, Result} ->
- etap:is(Result, Expect, "normal changes worked.")
- after 5000 ->
- io:format("never got the change", [])
- end.
-
-
-test_stream_once_timeout(Db) ->
- Self = self(),
- spawn(fun() ->
- Result = run_query(Db, [{since, 12},
- {stream, once},
- {timeout, 3000}]),
- Self ! {result, Result}
- end),
-
-
-
- Expect = {ok, 12, []},
-
- receive
- {result, Result} ->
- etap:is(Result, Expect, "got timeout.")
- after 5000 ->
- io:format("never got the change", [])
- end.
-
-test_stream_once_heartbeat(Db) ->
- Self = self(),
- spawn(fun() ->
- Result = run_query(Db, [{since, 12},
- {stream, once},
- {heartbeat, 1000}]),
- Self ! {result, Result}
- end),
-
- spawn(fun() ->
- timer:sleep(3000),
- {ok, Db1} = save_doc(Db, 12),
- couch_mrview:refresh(Db1, <<"_design/bar">>)
- end),
-
- Expect = {ok,13,[heartbeat,
- heartbeat,
- heartbeat,
- {{13,12,<<"12">>},12}]},
-
-
-
- receive
- {result, Result} ->
- etap:is(Result, Expect, "heartbeat OK.")
- after 5000 ->
- io:format("never got the change", [])
- end.
-
-
-test_stream(Db) ->
- Self = self(),
- spawn(fun() ->
- Result = run_query(Db, [{since, 13},
- stream,
- {timeout, 3000}]),
- Self ! {result, Result}
- end),
-
- spawn(fun() ->
- timer:sleep(1000),
- {ok, Db1} = save_doc(Db, 13),
- couch_mrview:refresh(Db1, <<"_design/bar">>),
- {ok, Db2} = save_doc(Db1, 14),
- couch_mrview:refresh(Db2, <<"_design/bar">>)
- end),
-
- Expect = {ok, 15,[{{14,13,<<"13">>},13},
- {{15,14,<<"14">>},14}]},
-
- receive
- {result, Result} ->
- etap:is(Result, Expect, "stream OK.")
- after 5000 ->
- io:format("never got the change", [])
- end.
-
-
-test_indexer(Db) ->
- Result = run_query(Db, [{since, 14}, refresh]),
- Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
- etap:is(Result, Expect, "refresh index by hand OK."),
-
- {ok, Db1} = save_doc(Db, 15),
- timer:sleep(1500),
- Result1 = run_query(Db1, [{since, 14}], false),
- Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
- {{16,15,<<"15">>},15}]},
- etap:is(Result1, Expect1, "changes indexed in background OK."),
- ok.
-
-
-save_doc(Db, Id) ->
- Doc = couch_mrview_test_util:doc(Id),
- {ok, _Rev} = couch_db:update_doc(Db, Doc, []),
- {ok, _} = couch_db:ensure_full_commit(Db),
- couch_db:reopen(Db).
-
-run_query(Db, Opts) ->
- run_query(Db, Opts, true).
-
-run_query(Db, Opts, Refresh) ->
- Fun = fun
- (stop, {LastSeq, Acc}) ->
- {ok, LastSeq, Acc};
- (heartbeat, Acc) ->
- {ok, [heartbeat | Acc]};
- (Event, Acc) ->
- {ok, [Event | Acc]}
- end,
- case Refresh of
- true ->
- couch_mrview:refresh(Db, <<"_design/bar">>);
- false ->
- ok
- end,
- {ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
- <<"baz">>, Fun, [], Opts),
- {ok, LastSeq, lists:reverse(R)}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/test/couch_mrview_changes_since_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_changes_since_tests.erl b/test/couch_mrview_changes_since_tests.erl
new file mode 100644
index 0000000..65f5a0b
--- /dev/null
+++ b/test/couch_mrview_changes_since_tests.erl
@@ -0,0 +1,166 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_changes_since_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(TIMEOUT, 1000).
+
+
+
+setup() ->
+ {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), changes),
+ Db.
+
+teardown(Db) ->
+ couch_db:close(Db),
+ couch_server:delete(Db#db.name, [?ADMIN_USER]),
+ ok.
+
+
+changes_since_test() ->
+ {
+ "changes_since tests",
+ {
+ setup,
+ fun test_util:start_couch/0, fun test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ test_basic/1,
+ test_range/1,
+ test_basic_since/1,
+ test_range_since/1,
+ test_basic_count/1,
+ test_range_count/1,
+ test_basic_count_since/1,
+ test_range_count_since/1,
+ test_compact/1,
+ test_remove_key/1
+ ]
+ }
+ }
+ }.
+
+test_basic(Db) ->
+ Result = run_query(Db, 0, []),
+ Expect = {ok, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+
+test_range(Db) ->
+ Result = run_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+ Expect = {ok, [
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+test_basic_since(Db) ->
+ Result = run_query(Db, 5, []),
+ Expect = {ok, [
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+test_range_since(Db) ->
+ Result = run_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+ Expect = {ok, [
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+test_basic_count(Db) ->
+ Result = run_count_query(Db, 0, []),
+ ?_assertEqual(Result, 10).
+
+test_range_count(Db) ->
+ Result = run_count_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+ ?_assertEqual(Result, 3).
+
+test_basic_count_since(Db) ->
+ Result = run_count_query(Db, 5, []),
+ ?_assertEqual(Result, 6).
+
+test_range_count_since(Db) ->
+ Result = run_count_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+ ?_assertEqual(Result, 2).
+
+test_compact(Db) ->
+ Result = couch_mrview:compact(Db, <<"_design/bar">>),
+ ?_assertEqual(Result, ok),
+ Count = run_count_query(Db, 0, []),
+ ?_assertEqual(Count, 10).
+
+test_remove_key(Db) ->
+ %% add new doc
+ Doc = couch_mrview_test_util:doc(11),
+ {ok, Rev} = couch_db:update_doc(Db, Doc, []),
+ RevStr = couch_doc:rev_to_str(Rev),
+ {ok, _} = couch_db:ensure_full_commit(Db),
+ {ok, Db1} = couch_db:reopen(Db),
+ Result = run_count_query(Db1, 0, []),
+ ?_assertEqual(Result, 11),
+ %% check new view key
+ Result1 = run_query(Db1, 0, [{start_key, 11}, {end_key, 11}]),
+ Expect = {ok, [
+ {{12, 11, <<"11">>}, 11}
+ ]},
+ ?_assertEqual(Result1, Expect),
+
+ %% delete doc
+ Doc2 = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"11">>},
+ {<<"_rev">>, RevStr},
+ {<<"_deleted">>, true}
+ ]}),
+ {ok, _} = couch_db:update_doc(Db1, Doc2, []),
+ {ok, Db2} = couch_db:reopen(Db1),
+ Result2 = run_count_query(Db2, 0, []),
+ ?_assertEqual(Result2, 11),
+ %% check new view key
+ Result3 = run_query(Db2, 0, [{start_key, 11}, {end_key, 11}]),
+ Expect2 = {ok, [
+ {{13, 11, <<"11">>}, {[{<<"_removed">>, true}]}}
+ ]},
+ ?_assertEqual(Result3, Expect2).
+
+run_query(Db, Since, Opts) ->
+ Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
+ {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+ Since, Fun, Opts, []),
+ {ok, lists:reverse(R)}.
+
+run_count_query(Db, Since, Opts) ->
+ couch_mrview:count_view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+ Since, Opts).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d6771df7/test/couch_mrview_index_changes_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_index_changes_tests.erl b/test/couch_mrview_index_changes_tests.erl
new file mode 100644
index 0000000..4a3a35c
--- /dev/null
+++ b/test/couch_mrview_index_changes_tests.erl
@@ -0,0 +1,217 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_all_docs_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+changes_index_test() ->
+ {
+ "changes index tests",
+ {
+ setup,
+ fun test_util:start_couch/0, fun test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun test_normal_changes/1,
+ fun test_stream_once/1,
+ fun test_stream_once_since/1,
+ fun test_stream_once_timeout/1,
+ fun test_stream_once_heartbeat/1,
+ fun test_stream/1,
+ fun test_indexer/1
+ ]
+ }
+ }
+ }.
+
+
+test_normal_changes(Db) ->
+ Result = run_query(Db, []),
+ Expect = {ok, 11, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+test_stream_once(Db) ->
+ Result = run_query(Db, [{stream, once}]),
+ Expect = {ok, 11, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ ?_assertEqual(Result, Expect).
+
+
+test_stream_once_since(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 11},
+ {stream, once}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(1000),
+ {ok, Db1} = save_doc(Db, 11),
+ couch_mrview:refresh(Db1, <<"_design/bar">>)
+ end),
+
+ Expect = {ok,12,[{{12,11,<<"11">>},11}]},
+
+ receive
+ {result, Result} ->
+ ?_assertEqual(Result, Expect)
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+test_stream_once_timeout(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 12},
+ {stream, once},
+ {timeout, 3000}]),
+ Self ! {result, Result}
+ end),
+
+
+
+ Expect = {ok, 12, []},
+
+ receive
+ {result, Result} ->
+ ?_assertEqual(Result, Expect)
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+test_stream_once_heartbeat(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 12},
+ {stream, once},
+ {heartbeat, 1000}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(3000),
+ {ok, Db1} = save_doc(Db, 12),
+ couch_mrview:refresh(Db1, <<"_design/bar">>)
+ end),
+
+ Expect = {ok,13,[heartbeat,
+ heartbeat,
+ heartbeat,
+ {{13,12,<<"12">>},12}]},
+
+
+
+ receive
+ {result, Result} ->
+ ?_assertEqual(Result, Expect)
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+test_stream(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 13},
+ stream,
+ {timeout, 3000}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(1000),
+ {ok, Db1} = save_doc(Db, 13),
+ couch_mrview:refresh(Db1, <<"_design/bar">>),
+ {ok, Db2} = save_doc(Db1, 14),
+ couch_mrview:refresh(Db2, <<"_design/bar">>)
+ end),
+
+ Expect = {ok, 15,[{{14,13,<<"13">>},13},
+ {{15,14,<<"14">>},14}]},
+
+ receive
+ {result, Result} ->
+ ?_assertEqual(Result, Expect)
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+test_indexer(Db) ->
+ Result = run_query(Db, [{since, 14}, refresh]),
+ Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
+ ?_assertEqual(Result, Expect),
+
+ {ok, Db1} = save_doc(Db, 15),
+ timer:sleep(1500),
+ Result1 = run_query(Db1, [{since, 14}], false),
+ Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
+ {{16,15,<<"15">>},15}]},
+ ?_assertEqual(Result1, Expect1),
+ ok.
+
+
+save_doc(Db, Id) ->
+ Doc = couch_mrview_test_util:doc(Id),
+ {ok, _Rev} = couch_db:update_doc(Db, Doc, []),
+ {ok, _} = couch_db:ensure_full_commit(Db),
+ couch_db:reopen(Db).
+
+run_query(Db, Opts) ->
+ run_query(Db, Opts, true).
+
+run_query(Db, Opts, Refresh) ->
+ Fun = fun
+ (stop, {LastSeq, Acc}) ->
+ {ok, LastSeq, Acc};
+ (heartbeat, Acc) ->
+ {ok, [heartbeat | Acc]};
+ (Event, Acc) ->
+ {ok, [Event | Acc]}
+ end,
+ case Refresh of
+ true ->
+ couch_mrview:refresh(Db, <<"_design/bar">>);
+ false ->
+ ok
+ end,
+ {ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
+ <<"baz">>, Fun, [], Opts),
+ {ok, LastSeq, lists:reverse(R)}.
[02/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: fix changes log
This changes make sure we pass correct keys to remove to the seq and
key_byseq btrees in views.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/19be43b6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/19be43b6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/19be43b6
Branch: refs/heads/master
Commit: 19be43b6bea268e02ef2859a9f7e36eb1c186d4d
Parents: deb8139
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 10:24:25 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:32 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/19be43b6/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 27cf795..f6557cf 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -311,13 +311,15 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
true ->
SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+ RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+ RemKSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
SKVs1 = SKVs ++ SToAdd,
{ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
- SKVs1, SToRem),
+ SKVs1, RemSKs),
{ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
couch_mrview_util:to_key_seq(SKVs1),
- couch_mrview_util:to_key_seq(SToRem)),
+ RemKSs),
{SBt, KSbt};
_ -> {nil, nil}
end,
@@ -382,8 +384,9 @@ update_log(Btree, Log, UpdatedSeq, _) ->
true ->
%% the log is updated, deleted old
%% record from the view
- DelAcc5 = dict:append(ViewId, {Seq, Key},
- DelAcc4),
+ DelAcc5 = dict:append(ViewId,
+ {Key, Seq, DocId},
+ DelAcc4),
{Log4, AddAcc4, DelAcc5};
false when Op /= del ->
%% an update operation has been
@@ -396,8 +399,9 @@ update_log(Btree, Log, UpdatedSeq, _) ->
{ViewId,
{Key,UpdatedSeq, del}},
Log4),
- DelAcc5 = dict:append(ViewId, {Seq, Key},
- DelAcc4),
+ DelAcc5 = dict:append(ViewId,
+ {Key, Seq, DocId},
+ DelAcc4),
AddAcc5 = dict:append(ViewId,
{{UpdatedSeq, Key},
{DocId, RemValue}},
[27/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Fix stuff which I broke in a rebase. I'll fix the git history later
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/487df675
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/487df675
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/487df675
Branch: refs/heads/master
Commit: 487df67528a14414b2048b4648b0d650a474945d
Parents: df3d28d
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Aug 28 23:40:59 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_index.erl | 14 +++++++++-----
src/couch_mrview_util.erl | 13 +++----------
2 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/487df675/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 63340cb..893ecd1 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -64,10 +64,14 @@ get(Property, State) ->
design_opts = Opts
} = State,
{ok, FileSize} = couch_file:bytes(Fd),
- {ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
- LogBtree,
- Views),
- ActiveSize = ExternalSize + couch_btree:size(Btree),
+ {ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
+ LogBtSize = case LogBtree of
+ nil ->
+ 0;
+ _ ->
+ couch_btree:size(LogBtree)
+ end,
+ ActiveSize = couch_btree:size(IdBtree) + LogBtSize + ExternalSize,
IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
@@ -183,7 +187,7 @@ finish_update(State) ->
commit(State) ->
Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
Resp = couch_file:write_header(State#mrst.fd, Header),
- couch_mrview_update_notifier:notify({index_update, State#mrst.db_name, State#mrst.idx_name}),
+ couch_event:notify(State#mrst.db_name, {index_update, State#mrst.idx_name}),
Resp.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/487df675/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index ce037d5..2850718 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -22,7 +22,7 @@
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
--export([calculate_external_size/3]).
+-export([calculate_external_size/1]).
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
-export([maybe_update_index_file/1]).
@@ -791,7 +791,7 @@ changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
-calculate_external_size(IdBt, LogBt, Views) ->
+calculate_external_size(Views) ->
SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
Size0 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
Size1 = case SBt of
@@ -803,14 +803,7 @@ calculate_external_size(IdBt, LogBt, Views) ->
_ -> sum_btree_sizes(Size1, couch_btree:size(KSBt))
end
end,
- Size = case LogBt of
- nil ->
- lists:foldl(SumFun, couch_btree:size(IdBt), Views);
- _ ->
- lists:foldl(SumFun, couch_btree:size(IdBt) +
- couch_btree:size(LogBt), Views)
- end,
- {ok, Size}.
+ {ok, lists:foldl(SumFun, 0, Views)}.
sum_btree_sizes(nil, _) ->
[30/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Make they key-by-key-and-seq view btree optional independently of the key-by-seq view btree
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/c0eedd42
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/c0eedd42
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/c0eedd42
Branch: refs/heads/master
Commit: c0eedd422415f36cf2d5bb68f7c72944fb373c79
Parents: 690d878
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Aug 26 15:28:56 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
include/couch_mrview.hrl | 2 +
src/couch_mrview.erl | 31 +++++-----
src/couch_mrview_compactor.erl | 30 ++++++----
src/couch_mrview_index.erl | 4 ++
src/couch_mrview_updater.erl | 71 ++++++++++++++---------
src/couch_mrview_util.erl | 111 ++++++++++++++++++++++++------------
6 files changed, 159 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index f1911db..b8ea08a 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -19,6 +19,7 @@
language,
design_opts=[],
seq_indexed=false,
+ keyseq_indexed=false,
lib,
views,
id_btree=nil,
@@ -45,6 +46,7 @@
seq_btree=nil,
key_byseq_btree=nil,
seq_indexed=false,
+ keyseq_indexed=false,
options=[]
}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index c6e413b..f6dec66 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -155,21 +155,22 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
Args0 = make_view_changes_args(Options),
{ok, {_, View, _}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
Args0),
- case View#mrview.seq_indexed of
- true ->
- OptList = make_view_changes_opts(StartSeq, Options, Args),
- Btree = case is_key_byseq(Options) of
- true -> View#mrview.key_byseq_btree;
- _ -> View#mrview.seq_btree
- end,
- AccOut = lists:foldl(fun(Opts, Acc0) ->
- {ok, _R, A} = couch_mrview_util:fold_changes(
- Btree, Fun, Acc0, Opts),
- A
- end, Acc, OptList),
- {ok, AccOut};
- _ ->
- {error, seqs_not_indexed}
+ #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
+ IsKSQuery = is_key_byseq(Options),
+ if (SIndexed andalso not IsKSQuery) orelse (KSIndexed andalso IsKSQuery) ->
+ OptList = make_view_changes_opts(StartSeq, Options, Args),
+ Btree = case IsKSQuery of
+ true -> View#mrview.key_byseq_btree;
+ _ -> View#mrview.seq_btree
+ end,
+ AccOut = lists:foldl(fun(Opts, Acc0) ->
+ {ok, _R, A} = couch_mrview_util:fold_changes(
+ Btree, Fun, Acc0, Opts),
+ A
+ end, Acc, OptList),
+ {ok, AccOut};
+ true ->
+ {error, seqs_not_indexed}
end.
count_view_changes_since(Db, DDoc, VName, SinceSeq) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index b6a8e78..6d9382f 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -42,6 +42,7 @@ compact(State) ->
id_btree=IdBtree,
log_btree=LogBtree,
seq_indexed=SeqIndexed,
+ keyseq_indexed=KeySeqIndexed,
views=Views
} = State,
erlang:put(io_priority, {view_compact, DbName, IdxName}),
@@ -63,7 +64,7 @@ compact(State) ->
views = EmptyViews
} = EmptyState,
- TotalChanges0 = case SeqIndexed of
+ TotalChanges0 = case SeqIndexed orelse KeySeqIndexed of
true -> NumDocIds * 2;
_ -> NumDocIds
end,
@@ -71,10 +72,10 @@ compact(State) ->
TotalChanges = lists:foldl(
fun(View, Acc) ->
{ok, Kvs} = couch_mrview_util:get_row_count(View),
- case SeqIndexed of
+ case SeqIndexed orelse KeySeqIndexed of
true ->
{ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
- Acc + Kvs + SKvs * 2;
+ Acc + Kvs + SKvs;
false ->
Acc + Kvs
end
@@ -187,18 +188,23 @@ compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
VID, BufferSize, Acc0),
%% are we indexing changes by sequences?
- {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+ {NewSeqBt, Acc2} = case View#mrview.seq_indexed of
true ->
- {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
- EmptyView#mrview.seq_btree,
- VID, BufferSize, Acc1),
- {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
- EmptyView#mrview.key_byseq_btree,
- VID, BufferSize, Acc2),
- {SBt, KSBt, Acc3};
+ compact_view_btree(View#mrview.seq_btree,
+ EmptyView#mrview.seq_btree,
+ VID, BufferSize, Acc1);
_ ->
- {nil, nil, Acc1}
+ {nil, Acc1}
end,
+ {NewKeyBySeqBt, FinalAcc} = case View#mrview.keyseq_indexed of
+ true ->
+ compact_view_btree(View#mrview.key_byseq_btree,
+ EmptyView#mrview.key_byseq_btree,
+ VID, BufferSize, Acc2);
+ _ ->
+ {nil, Acc2}
+ end,
+
{EmptyView#mrview{btree=NewBt,
seq_btree=NewSeqBt,
key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 0e6ef6f..63340cb 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -40,8 +40,10 @@ get(Property, State) ->
IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+ KeySeqIndexed = couch_util:get_value(<<"keyseq_indexed">>, Opts, false),
if IncDesign -> [include_design]; true -> [] end
++ if LocalSeq -> [local_seq]; true -> [] end
+ ++ if KeySeqIndexed -> [keyseq_indexed]; true -> [] end
++ if SeqIndexed -> [seq_indexed]; true -> [] end;
fd ->
State#mrst.fd;
@@ -70,9 +72,11 @@ get(Property, State) ->
IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+ KeySeqIndexed = couch_util:get_value(<<"keyseq_indexed">>, Opts, false),
UpdateOptions =
if IncDesign -> [<<"include_design">>]; true -> [] end
++ if LocalSeq -> [<<"local_seq">>]; true -> [] end
+ ++ if KeySeqIndexed -> [<<"keyseq_indexed">>]; true -> [] end
++ if SeqIndexed -> [<<"seq_indexed">>]; true -> [] end,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 3fe1dd6..2a67dff 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -86,27 +86,37 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
SeqsToRemove = lists:foldl(MakeDictFun, dict:new(), LLookups),
RemKeysFun = fun(#mrview{id_num=ViewId}=View) ->
+ #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
ToRem = couch_util:dict_find(ViewId, KeysToRemove, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, [], ToRem),
NewPurgeSeq = case VBtree2 =/= View#mrview.btree of
true -> PurgeSeq;
_ -> View#mrview.purge_seq
end,
- {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
- true ->
- SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+ {SeqBtree3, KeyBySeqBtree3} = if SIndexed orelse KSIndexed ->
+ SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+ {ok, SeqBtree2} = if SIndexed ->
SKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+ couch_btree:add_remove(View#mrview.seq_btree,
+ [], SKs);
+ true ->
+ {ok, nil}
+ end,
+ {ok, KeyBySeqBtree2} = if KSIndexed ->
KSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
- {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
- [], SKs),
- {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
- [], KSs),
- {SBt, KSbt};
- _ -> {nil, nil}
+ couch_btree:add_remove(View#mrview.key_byseq_btree,
+ [], KSs);
+ true ->
+ {ok, nil}
+ end,
+ {SeqBtree2, KeyBySeqBtree2};
+ true ->
+ {nil, nil}
end,
+
View#mrview{btree=VBtree2,
- seq_btree=SeqBtree2,
- key_byseq_btree=KeyBySeqBtree2,
+ seq_btree=SeqBtree3,
+ key_byseq_btree=KeyBySeqBtree3,
purge_seq=NewPurgeSeq}
end,
@@ -301,6 +311,7 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
end,
UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
+ #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
{ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
@@ -309,25 +320,33 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
end,
%% store the view changes.
- {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
- true ->
- SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
- SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+ {SeqBtree3, KeyBySeqBtree3} = if SIndexed orelse KSIndexed ->
+ SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+ SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+ SKVs1 = SKVs ++ SToAdd,
+
+ {ok, SeqBtree2} = if SIndexed ->
RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+ couch_btree:add_remove(View#mrview.seq_btree,
+ SKVs1, RemSKs);
+ true ->
+ {ok, nil}
+ end,
+ {ok, KeyBySeqBtree2} = if KSIndexed ->
RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
- SKVs1 = SKVs ++ SToAdd,
- {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
- SKVs1, RemSKs),
-
- {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
- couch_mrview_util:to_key_seq(SKVs1),
- RemKSs),
- {SBt, KSbt};
- _ -> {nil, nil}
+ couch_btree:add_remove(View#mrview.key_byseq_btree,
+ couch_mrview_util:to_key_seq(SKVs1),
+ RemKSs);
+ true ->
+ {ok, nil}
+ end,
+ {SeqBtree2, KeyBySeqBtree2};
+ true ->
+ {nil, nil}
end,
View#mrview{btree=VBtree2,
- seq_btree=SeqBtree2,
- key_byseq_btree=KeyBySeqBtree2,
+ seq_btree=SeqBtree3,
+ key_byseq_btree=KeyBySeqBtree3,
update_seq=NewUpdateSeq}
end,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0eedd42/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 7c639af..ce037d5 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -97,12 +97,13 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
end,
{DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}),
SeqIndexed = proplists:get_value(<<"seq_indexed">>, DesignOpts, false),
+ KeySeqIndexed = proplists:get_value(<<"keyseq_indexed">>, DesignOpts, false),
{RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
NumViews = fun({_, View}, N) ->
- {View#mrview{id_num=N, seq_indexed=SeqIndexed}, N+1}
+ {View#mrview{id_num=N, seq_indexed=SeqIndexed, keyseq_indexed=KeySeqIndexed}, N+1}
end,
{Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
@@ -116,7 +117,8 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
views=Views,
language=Language,
design_opts=DesignOpts,
- seq_indexed=SeqIndexed
+ seq_indexed=SeqIndexed,
+ keyseq_indexed=KeySeqIndexed
},
SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
{ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
@@ -161,7 +163,7 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
UpdateSeq = couch_db:get_update_seq(Db),
PurgeSeq = couch_db:get_purge_seq(Db),
Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
- State#mrst.seq_indexed}),
+ State#mrst.seq_indexed, State#mrst.keyseq_indexed}),
couch_index_util:hexsig(couch_util:md5(Bin));
view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
view_sig(Db, State, View, Args);
@@ -170,11 +172,12 @@ view_sig(_Db, State, View, Args0) ->
UpdateSeq = View#mrview.update_seq,
PurgeSeq = View#mrview.purge_seq,
SeqIndexed = View#mrview.seq_indexed,
+ KeySeqIndexed = View#mrview.keyseq_indexed,
Args = Args0#mrargs{
preflight_fun=undefined,
extra=[]
},
- Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, SeqIndexed, Args}),
+ Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}),
couch_index_util:hexsig(couch_util:md5(Bin)).
@@ -202,7 +205,12 @@ init_state(Db, Fd, State, #index_header{
view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates]
});
init_state(Db, Fd, State, Header) ->
- #mrst{language=Lang, views=Views, seq_indexed=SeqIndexed} = State,
+ #mrst{
+ language=Lang,
+ views=Views,
+ seq_indexed=SeqIndexed,
+ keyseq_indexed=KeySeqIndexed
+ } = State,
#mrheader{
seq=Seq,
purge_seq=PurgeSeq,
@@ -219,7 +227,7 @@ init_state(Db, Fd, State, Header) ->
IdBtOpts = [{compression, couch_db:compression(Db)}],
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
- {ok, LogBtree} = case SeqIndexed of
+ {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of
true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
false -> {ok, nil}
end,
@@ -266,20 +274,23 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
],
{ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
- {SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
- true ->
- BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
- ViewSeqBtOpts = [{less, fun less_json_seqs/2},
- {reduce, BySeqReduceFun},
- {compression, couch_db:compression(Db)}],
- KeyBySeqBtOpts = [{less, Less},
- {reduce, BySeqReduceFun},
- {compression, couch_db:compression(Db)}],
- {ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
- {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts),
- {SBt, KSBt};
- false ->
- {nil, nil}
+ BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
+ {ok, SeqBtree} = if View#mrview.seq_indexed ->
+ ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+ {reduce, BySeqReduceFun},
+ {compression, couch_db:compression(Db)}],
+
+ couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);
+ true ->
+ {ok, nil}
+ end,
+ {ok, KeyBySeqBtree} = if View#mrview.keyseq_indexed ->
+ KeyBySeqBtOpts = [{less, Less},
+ {reduce, BySeqReduceFun},
+ {compression, couch_db:compression(Db)}],
+ couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts);
+ true ->
+ {ok, nil}
end,
View#mrview{btree=Btree,
@@ -335,11 +346,25 @@ reduce_to_count(Reductions) ->
Count.
%% @doc get all changes for a view
-get_view_changes_count(#mrview{seq_btree=Btree}) ->
- couch_btree:fold_reduce(
- Btree, fun(_SeqStart, PartialReds, 0) ->
- {ok, couch_btree:final_reduce(Btree, PartialReds)}
- end,0, []).
+get_view_changes_count(View) ->
+ #mrview{seq_btree=SBtree, key_byseq_btree=KSBtree} = View,
+ CountFun = fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(SBtree, PartialReds)}
+ end,
+ {ok, Count} = case {SBtree, KSBtree} of
+ {nil, nil} ->
+ {ok, 0};
+ {#btree{}, nil} ->
+ couch_btree:fold_reduce(SBtree, CountFun, 0, []);
+ {nil, #btree{}} ->
+ couch_btree:fold_reduce(KSBtree, CountFun, 0, [])
+ end,
+ case {SBtree, KSBtree} of
+ {#btree{}, #btree{}} ->
+ {ok, Count*2};
+ _ ->
+ {ok, Count}
+ end.
fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
WrapperFun = fun(KV, Reds, Acc2) ->
@@ -563,11 +588,17 @@ make_header(State) ->
} = State,
ViewStates = lists:foldr(fun(V, Acc) ->
- {SeqBtState, KSeqBtState} = case V#mrview.seq_indexed of
+ SeqBtState = case V#mrview.seq_indexed of
+ true ->
+ couch_btree:get_state(V#mrview.seq_btree);
+ _ ->
+ nil
+ end,
+ KSeqBtState = case V#mrview.keyseq_indexed of
true ->
- {couch_btree:get_state(V#mrview.seq_btree),
- couch_btree:get_state(V#mrview.key_byseq_btree)};
- _ -> {nil, nil}
+ couch_btree:get_state(V#mrview.key_byseq_btree);
+ _ ->
+ nil
end,
[{couch_btree:get_state(V#mrview.btree),
SeqBtState,
@@ -642,12 +673,14 @@ reset_state(State) ->
fd=nil,
qserver=nil,
seq_indexed=State#mrst.seq_indexed,
+ keyseq_indexed=State#mrst.keyseq_indexed,
update_seq=0,
id_btree=nil,
log_btree=nil,
views=[View#mrview{btree=nil, seq_btree=nil,
key_byseq_btree=nil,
- seq_indexed=View#mrview.seq_indexed}
+ seq_indexed=View#mrview.seq_indexed,
+ keyseq_indexed=View#mrview.keyseq_indexed}
|| View <- State#mrst.views]
}.
@@ -757,14 +790,18 @@ changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
+
calculate_external_size(IdBt, LogBt, Views) ->
- SumFun = fun
- (#mrview{btree=Bt, seq_btree=nil}, Acc) ->
- sum_btree_sizes(Acc, couch_btree:size(Bt));
- (#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
- Acc1 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
- Acc2 = sum_btree_sizes(Acc1, couch_btree:size(SBt)),
- sum_btree_sizes(Acc2, couch_btree:size(KSBt))
+ SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
+ Size0 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+ Size1 = case SBt of
+ nil -> Size0;
+ _ -> sum_btree_sizes(Size0, couch_btree:size(SBt))
+ end,
+ case KSBt of
+ nil -> Size1;
+ _ -> sum_btree_sizes(Size1, couch_btree:size(KSBt))
+ end
end,
Size = case LogBt of
nil ->
[34/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
fix broken tests
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/8d3cdf16
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/8d3cdf16
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/8d3cdf16
Branch: refs/heads/master
Commit: 8d3cdf162db47ce87baa2238f658f8fe05724051
Parents: 5eab810
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Sep 9 09:52:49 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:34 2014 -0700
----------------------------------------------------------------------
test/couch_mrview_changes_since_tests.erl | 20 ++++++++++----------
test/couch_mrview_index_changes_tests.erl | 11 ++++++++++-
2 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/8d3cdf16/test/couch_mrview_changes_since_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_changes_since_tests.erl b/test/couch_mrview_changes_since_tests.erl
index 65f5a0b..69c9646 100644
--- a/test/couch_mrview_changes_since_tests.erl
+++ b/test/couch_mrview_changes_since_tests.erl
@@ -39,16 +39,16 @@ changes_since_test() ->
foreach,
fun setup/0, fun teardown/1,
[
- test_basic/1,
- test_range/1,
- test_basic_since/1,
- test_range_since/1,
- test_basic_count/1,
- test_range_count/1,
- test_basic_count_since/1,
- test_range_count_since/1,
- test_compact/1,
- test_remove_key/1
+ fun test_basic/1,
+ fun test_range/1,
+ fun test_basic_since/1,
+ fun test_range_since/1,
+ fun test_basic_count/1,
+ fun test_range_count/1,
+ fun test_basic_count_since/1,
+ fun test_range_count_since/1,
+ fun test_compact/1,
+ fun test_remove_key/1
]
}
}
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/8d3cdf16/test/couch_mrview_index_changes_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_mrview_index_changes_tests.erl b/test/couch_mrview_index_changes_tests.erl
index 4a3a35c..486deb5 100644
--- a/test/couch_mrview_index_changes_tests.erl
+++ b/test/couch_mrview_index_changes_tests.erl
@@ -10,12 +10,21 @@
% License for the specific language governing permissions and limitations under
% the License.
--module(couch_mrview_all_docs_tests).
+-module(couch_mrview_index_changes_tests).
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
+setup() ->
+ {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map),
+ Db.
+
+teardown(Db) ->
+ couch_db:close(Db),
+ couch_server:delete(Db#db.name, [?ADMIN_USER]),
+ ok.
+
changes_index_test() ->
{
"changes index tests",
[32/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Fix process_doc for temp views
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/5eab8107
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/5eab8107
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/5eab8107
Branch: refs/heads/master
Commit: 5eab8107b6671bc8a102bbb262ebb49ca442d7f0
Parents: fa7a888
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Sep 9 09:34:21 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:32 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/5eab8107/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 2dcce06..1cfa9c0 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -138,12 +138,16 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
{ok, State#mrst{doc_acc=[{nil, Seq, nil, nil} | Acc]}};
process_doc(#doc{id=Id, deleted=true}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
- {RevPos, [Rev | _]} = Doc#doc.revs,
- {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, deleted} | Acc]}};
+ Rev= extract_rev(Doc#doc.revs),
+ {ok, State#mrst{doc_acc=[{Id, Seq, Rev, deleted} | Acc]}};
process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
- {RevPos, [Rev | _]} = Doc#doc.revs,
- {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, Doc} | Acc]}}.
+ Rev = extract_rev(Doc#doc.revs),
+ {ok, State#mrst{doc_acc=[{Id, Seq, Rev, Doc} | Acc]}}.
+extract_rev({0, []}) ->
+ {0, []};
+extract_rev({RevPos, [Rev | _]}) ->
+ {RevPos, Rev}.
finish_update(#mrst{doc_acc=Acc}=State) ->
if Acc /= [] ->
[10/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: check removed keys from the index.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/96562222
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/96562222
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/96562222
Branch: refs/heads/master
Commit: 96562222256f1be7ec952f913aba8888628305eb
Parents: fb341ea
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 21:03:28 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
test/08-changes_since.t | 42 ++++++++++++++++++++++++++++++++++++++++--
1 file changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/96562222/test/08-changes_since.t
----------------------------------------------------------------------
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 3127cff..5a10f4c 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -15,7 +15,7 @@
% the License.
main(_) ->
- etap:plan(10),
+ etap:plan(14),
case (catch test()) of
ok ->
etap:end_tests();
@@ -40,7 +40,8 @@ test() ->
test_basic_count_since(Db),
test_range_count_since(Db),
test_compact(Db),
- test_util:stop_couch(),
+ test_remove_key(Db),
+ catch test_util:stop_couch(),
ok.
@@ -112,6 +113,43 @@ test_compact(Db) ->
Count = run_count_query(Db, 0, []),
etap:is(Count, 10, "compact view worked.").
+test_remove_key(Db) ->
+ %% add new doc
+ Doc = couch_mrview_test_util:doc(11),
+ {ok, Rev} = couch_db:update_doc(Db, Doc, []),
+ RevStr = couch_doc:rev_to_str(Rev),
+ {ok, _} = couch_db:ensure_full_commit(Db),
+ {ok, Db1} = couch_db:reopen(Db),
+ Result = run_count_query(Db1, 0, []),
+ etap:is(Result, 11, "Add new doc worked."),
+ %% check new view key
+ Result1 = run_query(Db1, 0, [{start_key, 11}, {end_key, 11}]),
+ Expect = {ok, [
+ {{12, 11, <<"11">>}, 11}
+ ]},
+ etap:is(Result1, Expect, "added key OK."),
+
+ %% delete doc
+ Doc2 = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"11">>},
+ {<<"_rev">>, RevStr},
+ {<<"_deleted">>, true}
+ ]}),
+ {ok, _} = couch_db:update_doc(Db1, Doc2, []),
+ {ok, Db2} = couch_db:reopen(Db1),
+ Result2 = run_count_query(Db2, 0, []),
+ etap:is(Result2, 11, "removed key saved."),
+ %% check new view key
+ Result3 = run_query(Db2, 0, [{start_key, 11}, {end_key, 11}]),
+ Expect2 = {ok, [
+ {{13, 11, <<"11">>}, {[{<<"_removed">>, true}]}}
+ ]},
+ etap:is(Result3, Expect2, "removed key OK.").
+
+
+
+
+
run_query(Db, Since, Opts) ->
Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
{ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
[41/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
remove unnecessary event type
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/4f925443
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/4f925443
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/4f925443
Branch: refs/heads/master
Commit: 4f925443fb3ffe2bcc82971cad1844f94af3ec4f
Parents: eb05dd7
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Sep 19 03:52:15 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:42 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_index.erl | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/4f925443/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 893ecd1..7113bb1 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -186,9 +186,7 @@ finish_update(State) ->
commit(State) ->
Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
- Resp = couch_file:write_header(State#mrst.fd, Header),
- couch_event:notify(State#mrst.db_name, {index_update, State#mrst.idx_name}),
- Resp.
+ couch_file:write_header(State#mrst.fd, Header).
compact(Db, State, Opts) ->
[20/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
extract couch_httpd changes API in its own module
Conflicts:
src/couch_mrview_updater.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/45da4bc7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/45da4bc7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/45da4bc7
Branch: refs/heads/master
Commit: 45da4bc7c8b694aeaf3c1e0dbe4c8d04b92e9eaf
Parents: 142a4c3
Author: benoitc <bc...@gmail.com>
Authored: Sun Feb 2 19:54:01 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/45da4bc7/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 7b24e42..16645aa 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -194,9 +194,6 @@ write_results(Parent, State) ->
Parent ! {new_state, NewState};
true ->
send_partial(NewState#mrst.partial_resp_pid, NewState),
- % notifify the view update
- couch_index_event:notify({index_update, {DbName, IdxName,
- couch_mrview_index}}),
write_results(Parent, NewState)
end
end.
[25/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: fix compaction with seqs_indexed=true
This change smakes sure to compact the view indexes when using the view
changes.
Conflicts:
src/couch_mrview_compactor.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/690d878b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/690d878b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/690d878b
Branch: refs/heads/master
Commit: 690d878b26ce73ce713febed8a2ece4b89ba5306
Parents: b3fef4c
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 15:29:10 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_compactor.erl | 84 +++++++++++++++++++++++++++++++++----
src/couch_mrview_util.erl | 12 +++++-
2 files changed, 88 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/690d878b/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 8b1c8f7..b6a8e78 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -40,6 +40,8 @@ compact(State) ->
sig=Sig,
update_seq=Seq,
id_btree=IdBtree,
+ log_btree=LogBtree,
+ seq_indexed=SeqIndexed,
views=Views
} = State,
erlang:put(io_priority, {view_compact, DbName, IdxName}),
@@ -57,15 +59,28 @@ compact(State) ->
#mrst{
id_btree = EmptyIdBtree,
+ log_btree = EmptyLogBtree,
views = EmptyViews
} = EmptyState,
+ TotalChanges0 = case SeqIndexed of
+ true -> NumDocIds * 2;
+ _ -> NumDocIds
+ end,
+
TotalChanges = lists:foldl(
fun(View, Acc) ->
{ok, Kvs} = couch_mrview_util:get_row_count(View),
- Acc + Kvs
+ case SeqIndexed of
+ true ->
+ {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
+ Acc + Kvs + SKvs * 2;
+ false ->
+ Acc + Kvs
+ end
end,
- NumDocIds, Views),
+ TotalChanges0, Views),
+
couch_task_status:add_task([
{type, view_compaction},
{database, DbName},
@@ -107,13 +122,25 @@ compact(State) ->
{ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+
+ {NewLogBtree, FinalAcc3} = case SeqIndexed of
+ true ->
+ compact_log(LogBtree, BufferSize,
+ FinalAcc2#acc{kvs=[],
+ kvs_size=0,
+ btree=EmptyLogBtree});
+ _ ->
+ {nil, FinalAcc2}
+ end,
+
{NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
compact_view(View, EmptyView, BufferSize, Acc)
- end, FinalAcc2, lists:zip(Views, EmptyViews)),
+ end, FinalAcc3, lists:zip(Views, EmptyViews)),
unlink(EmptyState#mrst.fd),
{ok, EmptyState#mrst{
id_btree=NewIdBtree,
+ log_btree=NewLogBtree,
views=NewViews,
update_seq=Seq
}}.
@@ -130,9 +157,53 @@ recompact(State) ->
{ok, State2}
end.
+compact_log(LogBtree, BufferSize, Acc0) ->
+ FoldFun = fun(KV, Acc) ->
+ #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+ KvsSize2 = KvsSize + ?term_size(KV),
+ case KvsSize2 >= BufferSize of
+ true ->
+ {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+ Acc2 = update_task(Acc, 1 + length(Kvs)),
+ {ok, Acc2#acc{
+ btree = Bt2, kvs = [], kvs_size = 0}};
+ _ ->
+ {ok, Acc#acc{
+ kvs = [KV | Kvs], kvs_size = KvsSize2}}
+ end
+ end,
+
+ {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0),
+ #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+ {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+ FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+ {NewLogBtree, FinalAcc2}.
%% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
+
+ {NewBt, Acc1} = compact_view_btree(View#mrview.btree,
+ EmptyView#mrview.btree,
+ VID, BufferSize, Acc0),
+
+ %% are we indexing changes by sequences?
+ {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+ true ->
+ {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
+ EmptyView#mrview.seq_btree,
+ VID, BufferSize, Acc1),
+ {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
+ EmptyView#mrview.key_byseq_btree,
+ VID, BufferSize, Acc2),
+ {SBt, KSBt, Acc3};
+ _ ->
+ {nil, nil, Acc1}
+ end,
+ {EmptyView#mrview{btree=NewBt,
+ seq_btree=NewSeqBt,
+ key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
+
+compact_view_btree(Btree, EmptyBtree, VID, BufferSize, Acc0) ->
Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
KvsSize2 = KvsSize + ?term_size(KV),
if KvsSize2 >= BufferSize ->
@@ -144,13 +215,12 @@ compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
end
end,
- InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
- {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
+ InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree},
+ {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc),
#acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
{ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
FinalAcc2 = update_task(VID, FinalAcc, length(Uncopied)),
- {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
-
+ {NewBt, FinalAcc2}.
update_task(Acc, ChangesInc) ->
update_task(null, Acc, ChangesInc).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/690d878b/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 734ad26..7c639af 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -18,6 +18,7 @@
-export([index_file/2, compaction_file/2, open_file/1]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_view_changes_count/1]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
@@ -333,6 +334,12 @@ reduce_to_count(Reductions) ->
{Count, _} = couch_btree:final_reduce(Reduce, Reductions),
Count.
+%% @doc get all changes for a view
+get_view_changes_count(#mrview{seq_btree=Btree}) ->
+ couch_btree:fold_reduce(
+ Btree, fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(Btree, PartialReds)}
+ end,0, []).
fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
WrapperFun = fun(KV, Reds, Acc2) ->
@@ -634,10 +641,13 @@ reset_state(State) ->
State#mrst{
fd=nil,
qserver=nil,
+ seq_indexed=State#mrst.seq_indexed,
update_seq=0,
id_btree=nil,
log_btree=nil,
- views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+ views=[View#mrview{btree=nil, seq_btree=nil,
+ key_byseq_btree=nil,
+ seq_indexed=View#mrview.seq_indexed}
|| View <- State#mrst.views]
}.
[36/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
make REM_VAL an atom in order to differentiate between possible emitted values and the key-removed placeholder.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/2e0be356
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/2e0be356
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/2e0be356
Branch: refs/heads/master
Commit: 2e0be3567c68968b24f1a233fb7b5e399e961ea8
Parents: b01a0bf
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Sep 16 12:27:26 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:41 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/2e0be356/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index f92a2a2..b7f91e1 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -17,7 +17,7 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
--define(REM_VAL, {[{<<"_removed">>, true}]}).
+-define(REM_VAL, removed).
start_update(Partial, State, NumChanges) ->
[16/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: add HTTP handler to retrieve the infos on a view.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/727ed624
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/727ed624
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/727ed624
Branch: refs/heads/master
Commit: 727ed6248428addfcf05c6c8951f636370084455
Parents: a043479
Author: benoitc <be...@apache.org>
Authored: Sat Feb 8 23:15:28 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 6 +++---
src/couch_mrview_http.erl | 12 ++++++++++++
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/727ed624/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 5b5ba25..4658781 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -216,11 +216,11 @@ get_view_info(Db, DDoc, VName) ->
{ok, TotalSeqs} = case SeqBtree of
nil -> {ok, 0};
_ ->
- {ok, {Count, _Reds}} = couch_btree:full_reduce(SeqBtree),
- {ok, Count}
+ couch_btree:full_reduce(SeqBtree)
end,
- {ok, [{update_seq, View#mrview.update_seq},
+ {ok, [{seq_indexed, View#mrview.seq_indexed},
+ {update_seq, View#mrview.update_seq},
{purge_seq, View#mrview.purge_seq},
{total_rows, TotalRows},
{total_seqs, TotalSeqs}]}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/727ed624/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index f3d3286..dfa294b 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -51,6 +51,18 @@ handle_all_docs_req(Req, _Db) ->
couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
+handle_view_req(#httpd{method='GET',
+ path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
+ Db, DDoc) ->
+
+ DDocId = <<"_design/", DDocName/binary >>,
+ {ok, Info} = couch_mrview:get_view_info(Db#db.name, DDocId, VName),
+
+ FinalInfo = [{db_name, Db#db.name},
+ {ddoc, DDocId},
+ {view, VName}] ++ Info,
+ couch_httpd:send_json(Req, 200, {FinalInfo});
+
handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
[_, _, _, _, ViewName] = Req#httpd.path_parts,
couch_stats:increment_counter([couchdb, httpd, view_reads]),
[33/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Make view sig backwards compatible
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/fa7a888f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/fa7a888f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/fa7a888f
Branch: refs/heads/master
Commit: fa7a888f52441355f3156514d113e2fcbb3750e6
Parents: d6771df
Author: Benjamin Bastian <be...@gmail.com>
Authored: Mon Sep 8 16:26:04 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:32 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_util.erl | 23 ++++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fa7a888f/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 2850718..6054e3a 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -162,9 +162,12 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
UpdateSeq = couch_db:get_update_seq(Db),
PurgeSeq = couch_db:get_purge_seq(Db),
- Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
- State#mrst.seq_indexed, State#mrst.keyseq_indexed}),
- couch_index_util:hexsig(couch_util:md5(Bin));
+ #mrst{
+ seq_indexed=SeqIndexed,
+ keyseq_indexed=KeySeqIndexed
+ } = State,
+ Term = view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed),
+ couch_index_util:hexsig(couch_util:md5(term_to_binary(Term)));
view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
view_sig(Db, State, View, Args);
view_sig(_Db, State, View, Args0) ->
@@ -177,8 +180,18 @@ view_sig(_Db, State, View, Args0) ->
preflight_fun=undefined,
extra=[]
},
- Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}),
- couch_index_util:hexsig(couch_util:md5(Bin)).
+ Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args),
+ couch_index_util:hexsig(couch_util:md5(term_to_binary(Term))).
+
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false) ->
+ {BaseSig, UpdateSeq, PurgeSeq};
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed) ->
+ {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed}.
+
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false, Args) ->
+ {BaseSig, UpdateSeq, PurgeSeq, Args};
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) ->
+ {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}.
init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
[06/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: add couch_mrview:count_view_changes_since/{4,5}
Add function to couch changes in a view index, similar to
couch_db:count_changes_since/2 .
Conflicts:
src/couch_mrview.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/8185ca7a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/8185ca7a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/8185ca7a
Branch: refs/heads/master
Commit: 8185ca7af804c40f292960d8c3c9687af117c167
Parents: 19be43b
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 11:51:11 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 33 +++++++++++++++++++++++++++++----
src/couch_mrview_util.erl | 8 ++++++--
2 files changed, 35 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/8185ca7a/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 343b91a..2f68fc0 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -16,6 +16,7 @@
-export([query_all_docs/2, query_all_docs/4]).
-export([query_view/3, query_view/4, query_view/6]).
-export([view_changes_since/6, view_changes_since/7]).
+-export([count_view_changes_since/4, count_view_changes_since/5]).
-export([get_info/2]).
-export([trigger_update/2, trigger_update/3]).
-export([compact/2, compact/3, cancel_compaction/2]).
@@ -160,10 +161,34 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
{error, seqs_not_indexed}
end.
-get_info(Db, DDocId) when is_binary(DDocId) ->
- DbName = mem3:dbname(Db#db.name),
- {ok, DDoc} = ddoc_cache:open(DbName, DDocId),
- get_info(Db, DDoc);
+count_view_changes_since(Db, DDoc, VName, SinceSeq) ->
+ count_view_changes_since(Db, DDoc, VName, SinceSeq, []).
+
+count_view_changes_since(Db, DDoc, VName, SinceSeq, Options) ->
+ Args0 = make_view_changes_args(Options),
+ {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+ Args0),
+ case View#mrview.seq_indexed of
+ true ->
+ OptList = make_view_changes_opts(SinceSeq, Options, Args),
+ Btree = case is_key_byseq(Options) of
+ true -> View#mrview.key_byseq_btree;
+ _ -> View#mrview.seq_btree
+ end,
+ lists:foldl(fun(Opts, Acc0) ->
+ {ok, N} = couch_btree:fold_reduce(
+ Btree, fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(
+ Btree, PartialReds)}
+ end,
+ 0, Opts),
+ Acc0 + N
+ end, 0, OptList);
+ _ ->
+ {error, seqs_not_indexed}
+ end.
+
+
get_info(Db, DDoc) ->
{ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
couch_index:get_info(Pid).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/8185ca7a/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index cd25887..6d48ed0 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -244,7 +244,6 @@ less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
Result < 0
end.
-
open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
ReduceFun =
@@ -273,10 +272,15 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
{SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
true ->
+ BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+ {reduce, BySeqReduceFun},
{compression, couch_db:compression(Db)}],
+ KeyBySeqBtOpts = [{less, Less},
+ {reduce, BySeqReduceFun},
+ {compression, couch_db:compression(Db)}],
{ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
- {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, ViewBtOpts),
+ {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts),
{SBt, KSBt};
false ->
{nil, nil}
[19/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_replicator: add replication using changes in a view
Instead of a database, the replicator can now filter the documents using
a view index. All documents having a key emitted in the view can be
replicated.
View parameters can be used. Which means that you can replicate results
corresponding to a key in a view or a range.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/d955e22a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/d955e22a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/d955e22a
Branch: refs/heads/master
Commit: d955e22a1fc15b0965cd3327abc69324f7602b17
Parents: 727ed62
Author: benoitc <be...@apache.org>
Authored: Sun Feb 9 00:43:23 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_http.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/d955e22a/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index dfa294b..fb2e4ac 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -53,7 +53,7 @@ handle_all_docs_req(Req, _Db) ->
handle_view_req(#httpd{method='GET',
path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
- Db, DDoc) ->
+ Db, _DDoc) ->
DDocId = <<"_design/", DDocName/binary >>,
{ok, Info} = couch_mrview:get_view_info(Db#db.name, DDocId, VName),
[23/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Add view filtering optimization to changes feeds. Remove unnecessary old stuff.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/24db15f3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/24db15f3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/24db15f3
Branch: refs/heads/master
Commit: 24db15f39015bec4710da97789a1728523ce92e1
Parents: cc45433
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Aug 22 23:57:48 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:35 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_changes.erl | 189 +-------------------------------------
src/couch_mrview_http.erl | 6 +-
2 files changed, 2 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/24db15f3/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index cb55655..ae5aa6e 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -12,194 +12,7 @@
%
-module(couch_mrview_changes).
--export([handle_changes/6, handle_view_changes/5]).
-
--include_lib("couch/include/couch_db.hrl").
-
--record(vst, {dbname,
- ddoc,
- view,
- view_options,
- since,
- callback,
- acc,
- user_timeout,
- timeout,
- heartbeat,
- timeout_acc=0,
- notifier,
- stream,
- refresh}).
-
--type changes_stream() :: true | false | once.
--type changes_options() :: [{stream, changes_stream()} |
- {since, integer()} |
- {view_options, list()} |
- {timeout, integer()} |
- {heartbeat, true | integer()} |
- {refresh, true | false}].
-
--export_type([changes_stream/0]).
--export_type([changes_options/0]).
+-export([handle_view_changes/5]).
handle_view_changes(Args, Req, Db, DDocId, ViewName) ->
-% couch_index_server:acquire_indexer(couch_mrview_index, Db#db.name, DDocId),
-% try
couch_changes:handle_changes(Args, Req, Db, {view, DDocId, ViewName}).
-% after
-% couch_index_server:release_indexer(couch_mrview_index, Db#db.name, DDocId)
-% end.
-
-
-
-%% @doc function returning changes in a streaming fashion if needed.
--spec handle_changes(binary(), binary(), binary(), function(), term(),
- changes_options()) -> ok | {error, term()}.
-handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
- Since = proplists:get_value(since, Options, 0),
- Stream = proplists:get_value(stream, Options, false),
- ViewOptions = proplists:get_value(view_options, Options, []),
- Refresh = proplists:get_value(refresh, Options, false),
-
- State0 = #vst{dbname=DbName,
- ddoc=DDocId,
- view=View,
- view_options=ViewOptions,
- since=Since,
- callback=Fun,
- acc=Acc},
-
- maybe_acquire_indexer(Refresh, DbName, DDocId),
- try
- case view_changes_since(State0) of
- {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
- case Stream of
- true ->
- start_loop(State#vst{stream=true}, Options);
- once when LastSeq =:= Since ->
- start_loop(State#vst{stream=once}, Options);
- _ ->
- Fun(stop, {LastSeq, Acc2})
- end;
- {stop, #vst{since=LastSeq, acc=Acc2}} ->
- Fun(stop, {LastSeq, Acc2});
- Error ->
- Error
- end
- after
- maybe_release_indexer(Refresh, DbName, DDocId)
- end.
-
-start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
- {UserTimeout, Timeout, Heartbeat} = changes_timeout(Options),
- Notifier = index_update_notifier(DbName, DDocId),
- try
- loop(State#vst{notifier=Notifier,
- user_timeout=UserTimeout,
- timeout=Timeout,
- heartbeat=Heartbeat})
- after
- couch_index_event:stop(Notifier)
- end.
-
-loop(#vst{since=Since, callback=Callback, acc=Acc,
- user_timeout=UserTimeout, timeout=Timeout,
- heartbeat=Heartbeat, timeout_acc=TimeoutAcc,
- stream=Stream}=State) ->
- receive
- index_update ->
- case view_changes_since(State) of
- {ok, State2} when Stream =:= true ->
- loop(State2#vst{timeout_acc=0});
- {ok, #vst{since=LastSeq, acc=Acc2}} ->
- Callback(stop, {LastSeq, Acc2});
- {stop, #vst{since=LastSeq, acc=Acc2}} ->
- Callback(stop, {LastSeq, Acc2})
- end;
- index_delete ->
- Callback(stop, {Since, Acc})
- after Timeout ->
- TimeoutAcc2 = TimeoutAcc + Timeout,
- case UserTimeout =< TimeoutAcc2 of
- true ->
- Callback(stop, {Since, Acc});
- false when Heartbeat =:= true ->
- case Callback(heartbeat, Acc) of
- {ok, Acc2} ->
- loop(State#vst{acc=Acc2, timeout_acc=TimeoutAcc2});
- {stop, Acc2} ->
- Callback(stop, {Since, Acc2})
- end;
- _ ->
- Callback(stop, {Since, Acc})
- end
- end.
-
-changes_timeout(Options) ->
- DefaultTimeout = list_to_integer(
- couch_config:get("httpd", "changes_timeout", "60000")
- ),
- UserTimeout = proplists:get_value(timeout, Options, DefaultTimeout),
- {Timeout, Heartbeat} = case proplists:get_value(heartbeat, Options) of
- undefined -> {UserTimeout, false};
- true ->
- T = erlang:min(DefaultTimeout, UserTimeout),
- {T, true};
- H ->
- T = erlang:min(H, UserTimeout),
- {T, true}
- end,
- {UserTimeout, Timeout, Heartbeat}.
-
-view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
- view_options=Options, since=Since,
- callback=Callback, acc=UserAcc}=State) ->
- Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {_Go, Acc2, OldSeq}) ->
- LastSeq = if OldSeq < Seq -> Seq;
- true -> OldSeq
- end,
-
- {Go, Acc3} = Callback(KV, Acc2),
- {Go, {Go, Acc3, LastSeq}}
- end,
-
- Acc0 = {ok, UserAcc, Since},
- case couch_mrview:view_changes_since(DbName, DDocId, View, Since,
- Wrapper, Options, Acc0) of
- {ok, {Go, UserAcc2, Since2}}->
- {Go, State#vst{since=Since2, acc=UserAcc2}};
- Error ->
- Error
- end.
-
-index_update_notifier(#db{name=DbName}, DDocId) ->
- index_update_notifier(DbName, DDocId);
-index_update_notifier(DbName, DDocId) ->
- Self = self(),
- {ok, NotifierPid} = couch_index_event:start_link(fun
- ({index_update, {Name, Id, couch_mrview_index}})
- when Name =:= DbName, Id =:= DDocId ->
- Self ! index_update;
- ({index_delete, {Name, Id, couch_mrview_index}})
- when Name =:= DbName, Id =:= DDocId ->
- Self ! index_delete;
- (_) ->
- ok
- end),
- NotifierPid.
-
-%% acquire the background indexing task so it can eventually be started
-%% if the process close the background task will be automatically
-%% released.
-maybe_acquire_indexer(false, _, _) ->
- ok;
-maybe_acquire_indexer(true, DbName, DDocId) ->
- couch_index_server:acquire_indexer(couch_mrview_index, DbName,
- DDocId).
-
-%% release the background indexing task so it can eventually be stopped
-maybe_release_indexer(false, _, _) ->
- ok;
-maybe_release_indexer(true, DbName, DDocId) ->
- couch_index_server:release_indexer(couch_mrview_index, DbName,
- DDocId).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/24db15f3/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 2b396b0..bb49583 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -36,10 +36,6 @@
check_view_etag/3
]).
--export([parse_boolean/1,
- parse_int/1,
- parse_pos_int/1]).
-
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -76,7 +72,7 @@ handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_cha
throw({bad_request, "view changes not enabled"})
end,
- ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
+ ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db),
ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
[18/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
use ERL_FLAGS instead of a shebang to launch tests
Conflicts:
test/03-red-views.t
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/a1623552
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/a1623552
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/a1623552
Branch: refs/heads/master
Commit: a1623552a84c3e66459a1c6afabec3f2ff89eca9
Parents: d955e22
Author: benoitc <bc...@gmail.com>
Authored: Thu Feb 13 22:59:47 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
test/08-changes_since.t | 1 -
test/09-index-events.t | 1 -
test/10-index-changes.t | 1 -
3 files changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a1623552/test/08-changes_since.t
----------------------------------------------------------------------
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 58c92e9..3107c1a 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -1,6 +1,5 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a1623552/test/09-index-events.t
----------------------------------------------------------------------
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 6cc1e9c..26d904d 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -1,6 +1,5 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a1623552/test/10-index-changes.t
----------------------------------------------------------------------
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index ce79eac..0c5e924 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -1,6 +1,5 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
[22/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Make feed=continuous work for view changes
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/c8394f90
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/c8394f90
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/c8394f90
Branch: refs/heads/master
Commit: c8394f90dd93210ea5a4a7acfb3b580cc0414bbe
Parents: 062c1e7
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Aug 22 17:46:13 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_http.erl | 2 +-
src/couch_mrview_index.erl | 4 ++-
src/couch_mrview_update_notifier.erl | 49 +++++++++++++++++++++++++++++++
3 files changed, 53 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c8394f90/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 1e54195..75c3cb5 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -64,7 +64,7 @@ handle_reindex_req(Req, _Db, _DDoc) ->
handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
- ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, DDocName, ViewName),
+ ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c8394f90/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 7598f13..0e6ef6f 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -178,7 +178,9 @@ finish_update(State) ->
commit(State) ->
Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
- couch_file:write_header(State#mrst.fd, Header).
+ Resp = couch_file:write_header(State#mrst.fd, Header),
+ couch_mrview_update_notifier:notify({index_update, State#mrst.db_name, State#mrst.idx_name}),
+ Resp.
compact(Db, State, Opts) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c8394f90/src/couch_mrview_update_notifier.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_update_notifier.erl b/src/couch_mrview_update_notifier.erl
new file mode 100644
index 0000000..1837b1f
--- /dev/null
+++ b/src/couch_mrview_update_notifier.erl
@@ -0,0 +1,49 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_update_notifier).
+
+-behaviour(gen_event).
+
+-export([start_link/1, notify/1]).
+-export([init/1, terminate/2, handle_event/2, handle_call/2, handle_info/2, code_change/3, stop/1]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+start_link(Exec) ->
+ couch_event_sup:start_link(couch_mrview_update, {couch_mrview_update_notifier, make_ref()}, Exec).
+
+notify(Event) ->
+ gen_event:notify(couch_mrview_update, Event).
+
+stop(Pid) ->
+ couch_event_sup:stop(Pid).
+
+init(Fun) ->
+ {ok, Fun}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+handle_event(Event, Fun) ->
+ Fun(Event),
+ {ok, Fun}.
+
+handle_call(_Request, State) ->
+ {reply, ok, State}.
+
+handle_info({'EXIT', Pid, Reason}, Pid) ->
+ ?LOG_ERROR("View update notification process ~p died: ~p", [Pid, Reason]),
+ remove_handler.
+
+code_change(_OldVsn, State, _Extra) ->
+ {ok, State}.
[38/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Fix seqs in mrview updater
This commit makes it so the correct seqs are inserted into the mrview
sequence tree. It also fixes the btree comparison function for that
tree.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/eb05dd78
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/eb05dd78
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/eb05dd78
Branch: refs/heads/master
Commit: eb05dd782437eaea32f198d78ae10bbccae2edd4
Parents: 11d4c17
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Sep 19 03:41:56 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:42 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 119 +++++++++++++++++++-------------------
src/couch_mrview_util.erl | 3 +-
2 files changed, 60 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/eb05dd78/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index c98c2f5..db5b5e9 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -206,8 +206,8 @@ write_results(Parent, State) ->
case accumulate_writes(State, State#mrst.write_queue, nil) of
stop ->
Parent ! {new_state, State};
- {Go, {Seq, ViewKVs, DocIdKeys, Log}} ->
- NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Log),
+ {Go, {Seq, ViewKVs, DocIdKeys, Seqs, Log}} ->
+ NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Seqs, Log),
if Go == stop ->
Parent ! {new_state, NewState};
true ->
@@ -229,17 +229,17 @@ start_query_server(State) ->
accumulate_writes(State, W, Acc0) ->
- {Seq, ViewKVs, DocIdKVs, Log} = case Acc0 of
- nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new()};
+ {Seq, ViewKVs, DocIdKVs, Seqs, Log} = case Acc0 of
+ nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new(), dict:new()};
_ -> Acc0
end,
case couch_work_queue:dequeue(W) of
closed when Seq == 0 ->
stop;
closed ->
- {stop, {Seq, ViewKVs, DocIdKVs, Log}};
+ {stop, {Seq, ViewKVs, DocIdKVs, Seqs, Log}};
{ok, Info} ->
- {_, _, NewIds, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Log),
+ {_, _, NewIds, _, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Seqs, Log),
case accumulate_more(length(NewIds)) of
true -> accumulate_writes(State, W, Acc);
false -> {ok, Acc}
@@ -256,29 +256,27 @@ accumulate_more(NumDocIds) ->
andalso CurrMem < list_to_integer(MinSize).
-merge_results([], SeqAcc, ViewKVs, DocIdKeys, Log) ->
- {SeqAcc, ViewKVs, DocIdKeys, Log};
-merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
- Fun = fun(RawResults, {VKV, DIK, Log2}) ->
- merge_results(RawResults, VKV, DIK, Log2)
+merge_results([], SeqAcc, ViewKVs, DocIdKeys, Seqs, Log) ->
+ {SeqAcc, ViewKVs, DocIdKeys, Seqs, Log};
+merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Seqs, Log) ->
+ Fun = fun(RawResults, {VKV, DIK, Seqs2, Log2}) ->
+ merge_results(RawResults, VKV, DIK, Seqs2, Log2)
end,
- {ViewKVs1, DocIdKeys1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Log},
- Results),
- merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1,
- Log1).
+ {ViewKVs1, DocIdKeys1, Seqs1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Seqs, Log}, Results),
+ merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1, Seqs1, Log1).
-merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
- {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
-merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
+merge_results({DocId, Seq, Rev, []}, ViewKVs, DocIdKeys, Seqs, Log) ->
+ {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, Seq, Seqs), dict:store({DocId, Rev}, [], Log)};
+merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Seqs, Log) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
case lists:flatten(Results) of
[] ->
- {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
+ {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, Seq, Seqs), dict:store({DocId, Rev}, [], Log)};
_ ->
{ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),
- {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}
+ {ViewKVs1, [ViewIdKeys | DocIdKeys], dict:store(DocId, Seq, Seqs), Log1}
end.
@@ -305,19 +303,25 @@ insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVA
[{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
-write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
+write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
#mrst{
id_btree=IdBtree,
log_btree=LogBtree,
first_build=FirstBuild
} = State,
+ Revs = dict:from_list(dict:fetch_keys(Log0)),
+
+ Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
+ dict:store(Id, DIKeys, Acc)
+ end, dict:new(), Log0),
+
{ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
ToRemByView = collapse_rem_keys(ToRemove, dict:new()),
{ok, SeqsToAdd, SeqsToRemove, LogBtree2} = case LogBtree of
nil -> {ok, undefined, undefined, nil};
- _ -> update_log(LogBtree, Log, UpdateSeq, FirstBuild)
+ _ -> update_log(LogBtree, Log, Revs, Seqs, FirstBuild)
end,
UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
@@ -336,12 +340,13 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
SKVs1 = SKVs ++ SToAdd,
{ok, SeqBtree2} = if SIndexed ->
- RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+ RemSKs = lists:sort([{Seq, Key} || {Key, Seq, _} <- SToRem]),
couch_btree:add_remove(View#mrview.seq_btree,
SKVs1, RemSKs);
true ->
{ok, nil}
end,
+
{ok, KeyBySeqBtree2} = if KSIndexed ->
RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
couch_btree:add_remove(View#mrview.key_byseq_btree,
@@ -367,7 +372,6 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
log_btree=LogBtree2
}.
-
update_id_btree(Btree, DocIdKeys, true) ->
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
couch_btree:query_modify(Btree, [], ToAdd, []);
@@ -377,64 +381,59 @@ update_id_btree(Btree, DocIdKeys, _) ->
ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
-update_log(Btree, Log, _UpdatedSeq, true) ->
- ToAdd = lists:flatmap(fun({{Id, _Rev}, Keys}) ->
- case Keys of
- [] ->
- [];
- _ ->
- [{Id, Keys}]
- end
- end, dict:to_list(Log)),
+
+update_log(Btree, Log, _Revs, _Seqs, true) ->
+ ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
+ DIKeys /= []],
{ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
{ok, dict:new(), dict:new(), LogBtree2};
-update_log(Btree, Log, UpdatedSeq, _) ->
+update_log(Btree, Log, Revs, Seqs, _) ->
%% build list of updated keys and Id
- Revs = dict:from_list(dict:fetch_keys(Log)),
- Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
- dict:store(Id, DIKeys, Acc)
- end, dict:new(), Log),
- {ToLook, Updated, Removed} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
- {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, Acc) ->
- {KeysAcc2, RemAcc2} = Acc,
- case Op of
- add ->
- {[{Id, ViewId, Key} | KeysAcc2], RemAcc2};
- del ->
- {KeysAcc2, [{Id, ViewId, Key} | RemAcc2]}
- end
- end, {KeysAcc, RemAcc}, DIKeys),
- {[Id | IdsAcc], KeysAcc1, RemAcc1}
- end, {[], [], []}, Log0),
+ {ToLook, Updated, Removed} = dict:fold(
+ fun(Id, [], {IdsAcc, KeysAcc, RemAcc}) ->
+ {[Id | IdsAcc], KeysAcc, RemAcc};
+ (Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
+ {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, {KeysAcc2, RemAcc2}) ->
+ case Op of
+ add -> {[{Id, ViewId, Key}|KeysAcc2], RemAcc2};
+ del -> {KeysAcc2, [{Id, ViewId, Key}|RemAcc2]}
+ end
+ end, {KeysAcc, RemAcc}, DIKeys),
+ {[Id | IdsAcc], KeysAcc1, RemAcc1}
+ end, {[], [], []}, Log),
MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
+
{Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
- lists:foldl(fun({ViewId, {Key, Seq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
- case lists:member({DocId, ViewId, Key}, Updated) of
+ lists:foldl(fun({ViewId, {Key, OldSeq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
+
+ IsUpdated = lists:member({DocId, ViewId, Key}, Updated),
+ IsRemoved = lists:member({DocId, ViewId, Key}, Removed),
+
+ case IsUpdated of
true ->
% the log is updated, deleted old record from the view
- DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+ DelAcc5 = dict:append(ViewId, {Key, OldSeq, DocId}, DelAcc4),
{Log4, AddAcc4, DelAcc5};
false ->
% an update operation has been logged for this key. We must
% now record it as deleted in the log, remove the old record
% in the view and update the view with a removed record.
- Log5 = case lists:member({DocId, ViewId, Key}, Removed) of
+ NewSeq = dict:fetch(DocId, Seqs),
+ Log5 = case IsRemoved of
false ->
- LogValue = {ViewId, {Key, UpdatedSeq, del}},
- dict:append(DocId, LogValue, Log4);
+ dict:append(DocId, {ViewId, {Key, NewSeq, del}}, Log4);
true ->
Log4
end,
- DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
Rev = dict:fetch(DocId, Revs),
- AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
- AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
+ DelAcc5 = dict:append(ViewId, {Key, OldSeq, DocId}, DelAcc4),
+ AddAcc5 = dict:append(ViewId, {{NewSeq, Key}, {DocId, ?REM_VAL, Rev}}, AddAcc4),
{Log5, AddAcc5, DelAcc5}
end
end, Acc, VIdKeys)
- end, {Log0, dict:new(), dict:new()}, KVsToLook),
+ end, {Log, dict:new(), dict:new()}, KVsToLook),
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
% store the new logs
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/eb05dd78/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 6054e3a..9669736 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -289,8 +289,7 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
{ok, SeqBtree} = if View#mrview.seq_indexed ->
- ViewSeqBtOpts = [{less, fun less_json_seqs/2},
- {reduce, BySeqReduceFun},
+ ViewSeqBtOpts = [{reduce, BySeqReduceFun},
{compression, couch_db:compression(Db)}],
couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);
[29/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Change function names to match changes in couchdb-couch
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/c31b0785
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/c31b0785
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/c31b0785
Branch: refs/heads/master
Commit: c31b07856af60db0873d53c469798fd35ea0ca69
Parents: c0eedd4
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Aug 28 00:02:48 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_http.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c31b0785/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index bb49583..fa933e7 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -72,9 +72,9 @@ handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_cha
throw({bad_request, "view changes not enabled"})
end,
- ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db),
+ ChangesArgs = couch_httpd_db:parse_changes_query(Req, Db),
ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
- couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
+ couch_httpd_db:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
handle_view_req(#httpd{method='GET',
[21/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Add preliminary version of view changes
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/062c1e7f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/062c1e7f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/062c1e7f
Branch: refs/heads/master
Commit: 062c1e7fc9b0cb96f7a95b2b218d4a473db809e4
Parents: 45da4bc
Author: Benjamin Bastian <be...@gmail.com>
Authored: Fri Aug 22 15:52:31 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 21 +++++++++++++++------
src/couch_mrview_changes.erl | 12 +++++++++++-
src/couch_mrview_http.erl | 7 +++++++
3 files changed, 33 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/062c1e7f/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 4658781..c6e413b 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -15,6 +15,7 @@
-export([validate/2]).
-export([query_all_docs/2, query_all_docs/4]).
-export([query_view/3, query_view/4, query_view/6]).
+-export([view_changes_since/5]).
-export([view_changes_since/6, view_changes_since/7]).
-export([count_view_changes_since/4, count_view_changes_since/5]).
-export([get_info/2]).
@@ -139,13 +140,21 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) ->
erlang:demonitor(Ref, [flush])
end.
+view_changes_since(View, StartSeq, Fun, Opts0, Acc) ->
+ Wrapper = fun(KV, _, Acc1) ->
+ Fun(KV, Acc1)
+ end,
+ Opts = [{start_key, {StartSeq + 1, <<>>}}] ++ Opts0,
+ {ok, _LastRed, AccOut} = couch_btree:fold(View#mrview.seq_btree, Wrapper, Acc, Opts),
+ {ok, AccOut}.
+
view_changes_since(Db, DDoc, VName, StartSeq, Fun, Acc) ->
view_changes_since(Db, DDoc, VName, StartSeq, Fun, [], Acc).
view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
Args0 = make_view_changes_args(Options),
- {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
- Args0),
+ {ok, {_, View, _}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+ Args0),
case View#mrview.seq_indexed of
true ->
OptList = make_view_changes_opts(StartSeq, Options, Args),
@@ -154,10 +163,10 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
_ -> View#mrview.seq_btree
end,
AccOut = lists:foldl(fun(Opts, Acc0) ->
- {ok, _R, A} = couch_mrview_util:fold_changes(
- Btree, Fun, Acc0, Opts),
- A
- end, Acc, OptList),
+ {ok, _R, A} = couch_mrview_util:fold_changes(
+ Btree, Fun, Acc0, Opts),
+ A
+ end, Acc, OptList),
{ok, AccOut};
_ ->
{error, seqs_not_indexed}
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/062c1e7f/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index 735ded8..cb55655 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -12,7 +12,7 @@
%
-module(couch_mrview_changes).
--export([handle_changes/6]).
+-export([handle_changes/6, handle_view_changes/5]).
-include_lib("couch/include/couch_db.hrl").
@@ -42,6 +42,16 @@
-export_type([changes_stream/0]).
-export_type([changes_options/0]).
+handle_view_changes(Args, Req, Db, DDocId, ViewName) ->
+% couch_index_server:acquire_indexer(couch_mrview_index, Db#db.name, DDocId),
+% try
+ couch_changes:handle_changes(Args, Req, Db, {view, DDocId, ViewName}).
+% after
+% couch_index_server:release_indexer(couch_mrview_index, Db#db.name, DDocId)
+% end.
+
+
+
%% @doc function returning changes in a streaming fashion if needed.
-spec handle_changes(binary(), binary(), binary(), function(), term(),
changes_options()) -> ok | {error, term()}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/062c1e7f/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 7a4c13c..1e54195 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -14,6 +14,7 @@
-export([
handle_all_docs_req/2,
+ handle_view_changes_req/3,
handle_reindex_req/3,
handle_view_req/3,
handle_temp_view_req/2,
@@ -61,6 +62,12 @@ handle_reindex_req(Req, _Db, _DDoc) ->
couch_httpd:send_method_not_allowed(Req, "POST").
+handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
+ ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
+ ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, DDocName, ViewName),
+ couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
+
+
handle_view_req(#httpd{method='GET',
path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
Db, _DDoc) ->
[15/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
add /<db>/_design/<ddocname>/_reindex handler
When you 'POST' on the _reindex handle all the views of the group will
be reindexed.
close #RCOUCH-36
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/142a4c3f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/142a4c3f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/142a4c3f
Branch: refs/heads/master
Commit: 142a4c3f99a28b83c753c29db4b4b27e8175ea4a
Parents: 42ff200
Author: benoitc <bc...@gmail.com>
Authored: Tue Mar 4 13:38:55 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:34 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_http.erl | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/142a4c3f/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index fb2e4ac..7a4c13c 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -14,6 +14,7 @@
-export([
handle_all_docs_req/2,
+ handle_reindex_req/3,
handle_view_req/3,
handle_temp_view_req/2,
handle_info_req/3,
@@ -50,6 +51,15 @@ handle_all_docs_req(#httpd{method='POST'}=Req, Db) ->
handle_all_docs_req(Req, _Db) ->
couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
+handle_reindex_req(#httpd{method='POST',
+ path_parts=[_, _, DName,<<"_reindex">>]}=Req,
+ Db, DDoc) ->
+ ok = couch_db:check_is_admin(Db),
+ couch_mrview:trigger_update(Db, <<"_design/", DName/binary>>),
+ couch_httpd:send_json(Req, 201, {[{<<"ok">>, true}]});
+handle_reindex_req(Req, _Db, _DDoc) ->
+ couch_httpd:send_method_not_allowed(Req, "POST").
+
handle_view_req(#httpd{method='GET',
path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
@@ -62,7 +72,6 @@ handle_view_req(#httpd{method='GET',
{ddoc, DDocId},
{view, VName}] ++ Info,
couch_httpd:send_json(Req, 200, {FinalInfo});
-
handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
[_, _, _, _, ViewName] = Req#httpd.path_parts,
couch_stats:increment_counter([couchdb, httpd, view_reads]),
[40/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Change log macro to couch_log call
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/e928480d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/e928480d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/e928480d
Branch: refs/heads/master
Commit: e928480d81c16d0959b6c913123c750fb6efbb0f
Parents: 4f92544
Author: Benjamin Bastian <be...@gmail.com>
Authored: Tue Oct 21 12:57:11 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:42 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_update_notifier.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/e928480d/src/couch_mrview_update_notifier.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_update_notifier.erl b/src/couch_mrview_update_notifier.erl
index 1837b1f..7c7ff1b 100644
--- a/src/couch_mrview_update_notifier.erl
+++ b/src/couch_mrview_update_notifier.erl
@@ -42,7 +42,7 @@ handle_call(_Request, State) ->
{reply, ok, State}.
handle_info({'EXIT', Pid, Reason}, Pid) ->
- ?LOG_ERROR("View update notification process ~p died: ~p", [Pid, Reason]),
+ couch_log:error("View update notification process ~p died: ~p", [Pid, Reason]),
remove_handler.
code_change(_OldVsn, State, _Extra) ->
[26/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Clean up update_log function for readability
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/c0ad453e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/c0ad453e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/c0ad453e
Branch: refs/heads/master
Commit: c0ad453e7c467557a2237e109d2c5af0eeab6f7d
Parents: c31b078
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Aug 28 15:29:22 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 109 ++++++++++++++------------------------
1 file changed, 39 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/c0ad453e/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 2a67dff..66db969 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -17,6 +17,8 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
+-define(REM_VAL, {[{<<"_removed">>, true}]}).
+
start_update(Partial, State, NumChanges) ->
QueueOpts = [{max_size, 100000}, {max_items, 500}],
@@ -367,83 +369,50 @@ update_id_btree(Btree, DocIdKeys, _) ->
ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
-walk_log(BTree, Fun, Acc, Ids) ->
- WrapFun = fun(KV, _Offset, Acc2) ->
- Fun(KV, Acc2)
- end,
- lists:foldl(fun(Id, Acc1) ->
- Opt = [{start_key, Id}, {end_key, Id}],
- {ok, _, A} = couch_btree:fold(BTree, WrapFun, Acc1, Opt),
- A
- end, Acc, Ids).
-
update_log(Btree, Log, _UpdatedSeq, true) ->
- ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
- DIKeys /= []],
+ ToAdd = lists:filter(fun({_, Keys}) -> Keys =/= [] end, dict:to_list(Log)),
{ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
{ok, dict:new(), dict:new(), LogBtree2};
update_log(Btree, Log, UpdatedSeq, _) ->
%% build list of updated keys and Id
- {ToLook, Updated} = dict:fold(fun
- (Id, [], {IdsAcc, KeysAcc}) ->
- {[Id | IdsAcc], KeysAcc};
- (Id, DIKeys, {IdsAcc, KeysAcc}) ->
- KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}},
- KeysAcc2) ->
- [{Id, ViewId, Key} | KeysAcc2]
- end, KeysAcc, DIKeys),
- {[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
-
- RemValue = {[{<<"_removed">>, true}]},
- {Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
- {Log2, AddAcc2, DelAcc2}) ->
-
- {Log3, AddAcc3, DelAcc3} = lists:foldl(fun({ViewId,{Key, Seq, Op}},
- {Log4, AddAcc4, DelAcc4}) ->
-
- case lists:member({DocId, ViewId, Key}, Updated) of
- true ->
- %% the log is updated, deleted old
- %% record from the view
- DelAcc5 = dict:append(ViewId,
- {Key, Seq, DocId},
- DelAcc4),
- {Log4, AddAcc4, DelAcc5};
- false when Op /= del ->
- %% an update operation has been
- %% logged for this key. We must now
- %% record it as deleted in the
- %% log, remove the old record in
- %% the view and update the view
- %% with a removed record.
- Log5 = dict:append(DocId,
- {ViewId,
- {Key,UpdatedSeq, del}},
- Log4),
- DelAcc5 = dict:append(ViewId,
- {Key, Seq, DocId},
- DelAcc4),
- AddAcc5 = dict:append(ViewId,
- {{UpdatedSeq, Key},
- {DocId, RemValue}},
- AddAcc4),
- {Log5, AddAcc5, DelAcc5};
- false ->
- %% the key has already been
- %% registered in the view as
- %% deleted, make sure to add it
- %% to the new log.
- Log5 = dict:append(DocId,
- {ViewId,
- {Key, Seq, del}}, Log4),
- {Log5, AddAcc4, DelAcc4}
- end
- end, {Log2, AddAcc2, DelAcc2}, VIdKeys),
- {ok, {Log3, AddAcc3, DelAcc3}}
- end, {Log, dict:new(), dict:new()}, ToLook),
+ {ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
+ KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
+ [{Id, ViewId, Key} | KeysAcc2]
+ end, KeysAcc, DIKeys),
+ {[Id | IdsAcc], KeysAcc1}
+ end, {[], []}, Log),
+
+ MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
+ KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
+ {Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
+ lists:foldl(fun({ViewId, {Key, Seq, Op}}, {Log4, AddAcc4, DelAcc4}) ->
+ case lists:member({DocId, ViewId, Key}, Updated) of
+ true ->
+ % the log is updated, deleted old record from the view
+ DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+ {Log4, AddAcc4, DelAcc5};
+ false when Op /= del ->
+ % an update operation has been logged for this key. We must
+ % now record it as deleted in the log, remove the old
+ % record in the view and update the view with a removed
+ % record.
+ LogValue = {ViewId, {Key, UpdatedSeq, del}},
+ Log5 = dict:append(DocId, LogValue, Log4),
+ DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+ AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL}},
+ AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
+ {Log5, AddAcc5, DelAcc5};
+ false ->
+ % the key has already been registered in the view as
+ % deleted, make sure to add it to the new log.
+ Log5 = dict:append(DocId, {ViewId, {Key, Seq, del}}, Log4),
+ {Log5, AddAcc4, DelAcc4}
+ end
+ end, Acc, VIdKeys)
+ end, {Log, dict:new(), dict:new()}, KVsToLook),
ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
- %% store the new logs
+ % store the new logs
{ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
{ok, AddAcc, DelAcc, LogBtree2}.
[35/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
fix first-pass log key addition and seq tree row removal
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/b01a0bf2
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/b01a0bf2
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/b01a0bf2
Branch: refs/heads/master
Commit: b01a0bf2d08ff357493f90effea8e42581db475b
Parents: 8d3cdf1
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Sep 11 12:21:45 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:41 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 52 ++++++++++++++++++++++++---------------
1 file changed, 32 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/b01a0bf2/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 1cfa9c0..f92a2a2 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -378,7 +378,14 @@ update_id_btree(Btree, DocIdKeys, _) ->
couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
update_log(Btree, Log, _UpdatedSeq, true) ->
- ToAdd = lists:filter(fun({_, Keys}) -> Keys =/= [] end, dict:to_list(Log)),
+ ToAdd = lists:flatmap(fun({{Id, _Rev}, Keys}) ->
+ case Keys of
+ [] ->
+ [];
+ _ ->
+ [{Id, Keys}]
+ end
+ end, dict:to_list(Log)),
{ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
{ok, dict:new(), dict:new(), LogBtree2};
update_log(Btree, Log, UpdatedSeq, _) ->
@@ -387,39 +394,44 @@ update_log(Btree, Log, UpdatedSeq, _) ->
Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
dict:store(Id, DIKeys, Acc)
end, dict:new(), Log),
- {ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
- KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
- [{Id, ViewId, Key} | KeysAcc2]
- end, KeysAcc, DIKeys),
- {[Id | IdsAcc], KeysAcc1}
- end, {[], []}, Log0),
+ {ToLook, Updated, Removed} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
+ {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, Acc) ->
+ {KeysAcc2, RemAcc2} = Acc,
+ case Op of
+ add ->
+ {[{Id, ViewId, Key} | KeysAcc2], RemAcc2};
+ del ->
+ {KeysAcc2, [{Id, ViewId, Key} | RemAcc2]}
+ end
+ end, {KeysAcc, RemAcc}, DIKeys),
+ {[Id | IdsAcc], KeysAcc1, RemAcc1}
+ end, {[], [], []}, Log0),
MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
{Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
- lists:foldl(fun({ViewId, {Key, Seq, Op}}, {Log4, AddAcc4, DelAcc4}) ->
+ lists:foldl(fun({ViewId, {Key, Seq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
case lists:member({DocId, ViewId, Key}, Updated) of
true ->
% the log is updated, deleted old record from the view
DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
{Log4, AddAcc4, DelAcc5};
- false when Op /= del ->
+ false ->
% an update operation has been logged for this key. We must
- % now record it as deleted in the log, remove the old
- % record in the view and update the view with a removed
- % record.
- LogValue = {ViewId, {Key, UpdatedSeq, del}},
- Log5 = dict:append(DocId, LogValue, Log4),
+ % now record it as deleted in the log, remove the old record
+ % in the view and update the view with a removed record.
+ Log5 = case lists:member({DocId, ViewId, Key}, Removed) of
+ false ->
+ LogValue = {ViewId, {Key, UpdatedSeq, del}},
+ dict:append(DocId, LogValue, Log4);
+ true ->
+ Log4
+ end,
DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
Rev = dict:fetch(DocId, Revs),
AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
- {Log5, AddAcc5, DelAcc5};
- false ->
- % the key has already been registered in the view as
- % deleted, make sure to add it to the new log.
- Log5 = dict:append(DocId, {ViewId, {Key, Seq, del}}, Log4),
- {Log5, AddAcc4, DelAcc4}
+ {Log5, AddAcc5, DelAcc5}
end
end, Acc, VIdKeys)
end, {Log0, dict:new(), dict:new()}, KVsToLook),
[11/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: add view changes test
test view changes and fix errors.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/fb341eaf
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/fb341eaf
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/fb341eaf
Branch: refs/heads/master
Commit: fb341eafdf877842bbf536d31a7a53f55a6f6345
Parents: 8185ca7
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 17:24:54 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_test_util.erl | 31 +++++++--
src/couch_mrview_updater.erl | 2 +-
src/couch_mrview_util.erl | 34 ++++++----
test/08-changes_since.t | 123 ++++++++++++++++++++++++++++++++++++
4 files changed, 168 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fb341eaf/src/couch_mrview_test_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index 40ccf09..c68010c 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -40,14 +40,31 @@ save_docs(Db, Docs) ->
make_docs(Count) ->
- make_docs(Count, []).
-
-make_docs(Count, Acc) when Count =< 0 ->
- Acc;
-make_docs(Count, Acc) ->
- make_docs(Count-1, [doc(Count) | Acc]).
-
+ [doc(I) || I <- lists:seq(1, Count)].
+ddoc(changes) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/bar">>},
+ {<<"options">>, {[
+ {<<"seq_indexed">>, true}
+ ]}},
+ {<<"views">>, {[
+ {<<"baz">>, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>}
+ ]}},
+ {<<"bing">>, {[
+ {<<"map">>, <<"function(doc) {}">>}
+ ]}},
+ {<<"zing">>, {[
+ {<<"map">>, <<
+ "function(doc) {\n"
+ " if(doc.foo !== undefined)\n"
+ " emit(doc.foo, 0);\n"
+ "}"
+ >>}
+ ]}}
+ ]}}
+ ]});
ddoc(map) ->
couch_doc:from_json_obj({[
{<<"_id">>, <<"_design/bar">>},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fb341eaf/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index f6557cf..16645aa 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -312,7 +312,7 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
- RemKSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
+ RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
SKVs1 = SKVs ++ SToAdd,
{ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
SKVs1, RemSKs),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fb341eaf/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 6d48ed0..f5fd66c 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -236,13 +236,8 @@ init_state(Db, Fd, State, Header) ->
views=Views2
}.
-less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
- case couch_ejson_compare:less(SeqA, SeqB) of
- 0 ->
- couch_ejson_compare:less_json(JsonA, JsonB);
- Result ->
- Result < 0
- end.
+less_json_seqs({SeqA, _JsonA}, {SeqB, _JsonB}) ->
+ couch_ejson_compare:less(SeqA, SeqB) < 0.
open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
@@ -355,12 +350,23 @@ fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
{stop, Acc2}
end.
+
fold_changes(Bt, Fun, Acc, Opts) ->
WrapperFun = fun(KV, _Reds, Acc2) ->
- Fun(changes_expand_dups([KV], []), Acc2)
+ fold_changes_fun(Fun, changes_expand_dups([KV], []), Acc2)
end,
{ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
+fold_changes_fun(_Fun, [], Acc) ->
+ {ok, Acc};
+fold_changes_fun(Fun, [KV|Rest], Acc) ->
+ case Fun(KV, Acc) of
+ {ok, Acc2} ->
+ fold_changes_fun(Fun, Rest, Acc2);
+ {stop, Acc2} ->
+ {stop, Acc2}
+ end.
+
fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) ->
#mrview{
@@ -788,15 +794,15 @@ expand_dups([KV | Rest], Acc) ->
changes_expand_dups([], Acc) ->
lists:reverse(Acc);
changes_expand_dups([{{[Key, Seq], DocId}, {dups, Vals}} | Rest], Acc) ->
- Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+ Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
changes_expand_dups(Rest, Expanded ++ Acc);
-changes_expand_dups([{{Key, Seq}, {DocId, {dups, Vals}}} | Rest], Acc) ->
- Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+changes_expand_dups([{{Seq, Key}, {DocId, {dups, Vals}}} | Rest], Acc) ->
+ Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
changes_expand_dups(Rest, Expanded ++ Acc);
changes_expand_dups([{{[Key, Seq], DocId}, Val} | Rest], Acc) ->
- changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]);
-changes_expand_dups([{{Key, Seq}, {DocId, Val}} | Rest], Acc) ->
- changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]).
+ changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]);
+changes_expand_dups([{{Seq, Key}, {DocId, Val}} | Rest], Acc) ->
+ changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]).
maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
[];
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fb341eaf/test/08-changes_since.t
----------------------------------------------------------------------
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
new file mode 100644
index 0000000..3127cff
--- /dev/null
+++ b/test/08-changes_since.t
@@ -0,0 +1,123 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+ etap:plan(10),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ timer:sleep(300),
+ ok.
+
+test() ->
+ test_util:start_couch(),
+
+ {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+
+ test_basic(Db),
+ test_range(Db),
+ test_basic_since(Db),
+ test_range_since(Db),
+ test_basic_count(Db),
+ test_range_count(Db),
+ test_basic_count_since(Db),
+ test_range_count_since(Db),
+ test_compact(Db),
+ test_util:stop_couch(),
+ ok.
+
+
+test_basic(Db) ->
+ Result = run_query(Db, 0, []),
+ Expect = {ok, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ etap:is(Result, Expect, "Simple view query worked.").
+
+
+test_range(Db) ->
+ Result = run_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+ Expect = {ok, [
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5}
+ ]},
+ etap:is(Result, Expect, "Query with range works.").
+
+test_basic_since(Db) ->
+ Result = run_query(Db, 5, []),
+ Expect = {ok, [
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ etap:is(Result, Expect, "Simple view query since 5 worked.").
+
+test_range_since(Db) ->
+ Result = run_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+ Expect = {ok, [
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5}
+ ]},
+ etap:is(Result, Expect, "Query with range since 5 works.").
+
+test_basic_count(Db) ->
+ Result = run_count_query(Db, 0, []),
+ etap:is(Result, 10, "Simple view count worked.").
+
+test_range_count(Db) ->
+ Result = run_count_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+ etap:is(Result, 3, "Count with range works.").
+
+test_basic_count_since(Db) ->
+ Result = run_count_query(Db, 5, []),
+ etap:is(Result, 6, "Simple view count since 5 worked.").
+
+test_range_count_since(Db) ->
+ Result = run_count_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+ etap:is(Result, 2, "Count with range since 5 works.").
+
+test_compact(Db) ->
+ Result = couch_mrview:compact(Db, <<"_design/bar">>),
+ etap:is(Result, ok, "compact view is OK"),
+ Count = run_count_query(Db, 0, []),
+ etap:is(Count, 10, "compact view worked.").
+
+run_query(Db, Since, Opts) ->
+ Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
+ {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+ Since, Fun, Opts, []),
+ {ok, lists:reverse(R)}.
+
+run_count_query(Db, Since, Opts) ->
+ couch_mrview:count_view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+ Since, Opts).
[09/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: couch_mrview_changes:handle_changes
Similar to couch_changes:handle_changes but for view changes. It add
support for longpolling, normal and continuous stream The API differs
from the one for doc by beeing independant from the transport: the
support of HTTP will be added on top for example.
This API will be also used to replace the view filter in the current _changes
API.
Also add unittests.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/1c24c425
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/1c24c425
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/1c24c425
Branch: refs/heads/master
Commit: 1c24c425f2ec9fa63b0e01a13673af234043ee30
Parents: 18b5f6f
Author: benoitc <be...@apache.org>
Authored: Fri Jan 31 13:13:23 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_changes.erl | 173 ++++++++++++++++++++++++++++++++
src/couch_mrview_test_util.erl | 2 +
test/09-index-events.t | 17 +++-
test/10-index-changes.t | 194 ++++++++++++++++++++++++++++++++++++
4 files changed, 385 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/1c24c425/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
new file mode 100644
index 0000000..2b8f910
--- /dev/null
+++ b/src/couch_mrview_changes.erl
@@ -0,0 +1,173 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+%
+-module(couch_mrview_changes).
+
+-export([handle_changes/6]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+-record(vst, {dbname,
+ ddoc,
+ view,
+ view_options,
+ since,
+ callback,
+ acc,
+ user_timeout,
+ timeout,
+ heartbeat,
+ timeout_acc=0,
+ notifier,
+ stream}).
+
+-type changes_stream() :: true | false | once.
+-type changes_options() :: [{stream, changes_stream()} |
+ {since, integer()} |
+ {view_options, list()} |
+ {timeout, integer()} |
+ {heartbeat, true | integer()}].
+
+-export_type([changes_stream/0]).
+-export_type([changes_options/0]).
+
+%% @doc function returning changes in a streaming fashion if needed.
+-spec handle_changes(binary(), binary(), binary(), function(), term(),
+ changes_options()) -> ok | {error, term()}.
+handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
+ Since = proplists:get_value(since, Options, 0),
+ Stream = proplists:get_value(stream, Options, false),
+ ViewOptions = proplists:get_value(view_options, Options, []),
+
+ State0 = #vst{dbname=DbName,
+ ddoc=DDocId,
+ view=View,
+ view_options=ViewOptions,
+ since=Since,
+ callback=Fun,
+ acc=Acc},
+
+ case view_changes_since(State0) of
+ {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
+ case Stream of
+ true ->
+ start_loop(State#vst{stream=true}, Options);
+ once when LastSeq =:= Since ->
+ start_loop(State#vst{stream=once}, Options);
+ _ ->
+ Fun(stop, {LastSeq, Acc2})
+ end;
+ {stop, #vst{since=LastSeq, acc=Acc2}} ->
+ Fun(stop, {LastSeq, Acc2});
+ Error ->
+ Error
+ end.
+
+start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
+ {UserTimeout, Timeout, Heartbeat} = changes_timeout(Options),
+ Notifier = index_update_notifier(DbName, DDocId),
+ try
+ loop(State#vst{notifier=Notifier,
+ user_timeout=UserTimeout,
+ timeout=Timeout,
+ heartbeat=Heartbeat})
+ after
+ couch_index_event:stop(Notifier)
+ end.
+
+loop(#vst{since=Since, callback=Callback, acc=Acc,
+ user_timeout=UserTimeout, timeout=Timeout,
+ heartbeat=Heartbeat, timeout_acc=TimeoutAcc,
+ stream=Stream}=State) ->
+ receive
+ index_update ->
+ case view_changes_since(State) of
+ {ok, State2} when Stream =:= true ->
+ loop(State2#vst{timeout_acc=0});
+ {ok, #vst{since=LastSeq, acc=Acc2}} ->
+ Callback(stop, {LastSeq, Acc2});
+ {stop, #vst{since=LastSeq, acc=Acc2}} ->
+ Callback(stop, {LastSeq, Acc2})
+ end;
+ index_delete ->
+ Callback(stop, {Since, Acc})
+ after Timeout ->
+ TimeoutAcc2 = TimeoutAcc + Timeout,
+ case UserTimeout =< TimeoutAcc2 of
+ true ->
+ Callback(stop, {Since, Acc});
+ false when Heartbeat =:= true ->
+ case Callback(heartbeat, Acc) of
+ {ok, Acc2} ->
+ loop(State#vst{acc=Acc2, timeout_acc=TimeoutAcc2});
+ {stop, Acc2} ->
+ Callback(stop, {Since, Acc2})
+ end;
+ _ ->
+ Callback(stop, {Since, Acc})
+ end
+ end.
+
+changes_timeout(Options) ->
+ DefaultTimeout = list_to_integer(
+ couch_config:get("httpd", "changes_timeout", "60000")
+ ),
+ UserTimeout = proplists:get_value(timeout, Options, DefaultTimeout),
+ {Timeout, Heartbeat} = case proplists:get_value(heartbeat, Options) of
+ undefined -> {UserTimeout, false};
+ true ->
+ T = erlang:min(DefaultTimeout, UserTimeout),
+ {T, true};
+ H ->
+ T = erlang:min(H, UserTimeout),
+ {T, true}
+ end,
+ {UserTimeout, Timeout, Heartbeat}.
+
+view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
+ view_options=Options, since=Since,
+ callback=Callback, acc=UserAcc}=State) ->
+ Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {Go, Acc2, OldSeq}) ->
+ LastSeq = if OldSeq < Seq -> Seq;
+ true -> OldSeq
+ end,
+
+ case Callback(KV, Acc2) of
+ {ok, Acc3} -> {ok, {Go, Acc3, LastSeq}};
+ {stop, Acc3} -> {stop, {stop, Acc3, LastSeq}}
+ end
+ end,
+
+ Acc0 = {ok, UserAcc, Since},
+ case couch_mrview:view_changes_since(DbName, DDocId, View, Since,
+ Wrapper, Options, Acc0) of
+ {ok, {Go, UserAcc2, Since2}}->
+ {Go, State#vst{since=Since2, acc=UserAcc2}};
+ Error ->
+ Error
+ end.
+
+index_update_notifier(#db{name=DbName}, DDocId) ->
+ index_update_notifier(DbName, DDocId);
+index_update_notifier(DbName, DDocId) ->
+ Self = self(),
+ {ok, NotifierPid} = couch_index_event:start_link(fun
+ ({index_update, {Name, Id, couch_mrview_index}})
+ when Name =:= DbName, Id =:= DDocId ->
+ Self ! index_update;
+ ({index_delete, {Name, Id, couch_mrview_index}})
+ when Name =:= DbName, Id =:= DDocId ->
+ Self ! index_delete;
+ (_) ->
+ ok
+ end),
+ NotifierPid.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/1c24c425/src/couch_mrview_test_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index c68010c..1d3d788 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -33,6 +33,8 @@ new_db(Name, Type) ->
{ok, Db} = couch_db:create(Name, [?ADMIN_USER]),
save_docs(Db, [ddoc(Type)]).
+delete_db(Name) ->
+ couch_server:delete(Name, [{user_ctx, ?ADMIN}]).
save_docs(Db, Docs) ->
{ok, _} = couch_db:update_docs(Db, Docs, []),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/1c24c425/test/09-index-events.t
----------------------------------------------------------------------
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 90654b8..1489e4e 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -15,7 +15,7 @@
% the License.
main(_) ->
- etap:plan(2),
+ etap:plan(4),
case (catch test()) of
ok ->
etap:end_tests();
@@ -30,6 +30,7 @@ test() ->
test_util:start_couch(),
{ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
test_update_event(Db),
+ test_delete_event(Db),
test_util:stop_couch(),
ok.
@@ -44,3 +45,17 @@ test_update_event(Db) ->
etap:is(Event, Expect, "index update events OK")
end,
couch_index_event:stop(Pid).
+
+test_delete_event(Db) ->
+ ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+ {ok, Pid} = couch_index_event:start_link(self()),
+
+ etap:ok(is_pid(Pid), "event handler added"),
+ couch_mrview_test_util:delete_db(<<"foo">>),
+ Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
+ couch_mrview_index}},
+ receive
+ Event ->
+ etap:is(Event, Expect, "index delete events OK")
+ end,
+ couch_index_event:stop(Pid).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/1c24c425/test/10-index-changes.t
----------------------------------------------------------------------
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
new file mode 100644
index 0000000..627376f
--- /dev/null
+++ b/test/10-index-changes.t
@@ -0,0 +1,194 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+ etap:plan(6),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ timer:sleep(300),
+ ok.
+
+test() ->
+ test_util:start_couch(),
+ {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+ test_normal_changes(Db),
+ test_stream_once(Db),
+ test_stream_once_since(Db),
+ test_stream_once_timeout(Db),
+ test_stream_once_heartbeat(Db),
+ test_stream(Db),
+ test_util:stop_couch(),
+ ok.
+
+test_normal_changes(Db) ->
+ Result = run_query(Db, []),
+ Expect = {ok, 11, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ etap:is(Result, Expect, "normal changes worked.").
+
+test_stream_once(Db) ->
+ Result = run_query(Db, [{stream, once}]),
+ Expect = {ok, 11, [
+ {{2, 1, <<"1">>}, 1},
+ {{3, 10, <<"10">>}, 10},
+ {{4, 2, <<"2">>}, 2},
+ {{5, 3, <<"3">>}, 3},
+ {{6, 4, <<"4">>}, 4},
+ {{7, 5, <<"5">>}, 5},
+ {{8, 6, <<"6">>}, 6},
+ {{9, 7, <<"7">>}, 7},
+ {{10, 8, <<"8">>}, 8},
+ {{11, 9, <<"9">>}, 9}
+ ]},
+ etap:is(Result, Expect, "stream once since 0 worked.").
+
+
+test_stream_once_since(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 11},
+ {stream, once}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(1000),
+ {ok, Db1} = save_doc(Db, 11),
+ couch_mrview:refresh(Db1, <<"_design/bar">>)
+ end),
+
+ Expect = {ok,12,[{{12,11,<<"11">>},11}]},
+
+ receive
+ {result, Result} ->
+ etap:is(Result, Expect, "normal changes worked.")
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+test_stream_once_timeout(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 12},
+ {stream, once},
+ {timeout, 3000}]),
+ Self ! {result, Result}
+ end),
+
+
+
+ Expect = {ok, 12, []},
+
+ receive
+ {result, Result} ->
+ etap:is(Result, Expect, "got timeout.")
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+test_stream_once_heartbeat(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 12},
+ {stream, once},
+ {heartbeat, 1000}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(3000),
+ {ok, Db1} = save_doc(Db, 12),
+ couch_mrview:refresh(Db1, <<"_design/bar">>)
+ end),
+
+ Expect = {ok,13,[heartbeat,
+ heartbeat,
+ heartbeat,
+ {{13,12,<<"12">>},12}]},
+
+
+
+ receive
+ {result, Result} ->
+ etap:is(Result, Expect, "heartbeat OK.")
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+test_stream(Db) ->
+ Self = self(),
+ spawn(fun() ->
+ Result = run_query(Db, [{since, 13},
+ stream,
+ {timeout, 3000}]),
+ Self ! {result, Result}
+ end),
+
+ spawn(fun() ->
+ timer:sleep(1000),
+ {ok, Db1} = save_doc(Db, 13),
+ couch_mrview:refresh(Db1, <<"_design/bar">>),
+ {ok, Db2} = save_doc(Db1, 14),
+ couch_mrview:refresh(Db2, <<"_design/bar">>)
+ end),
+
+ Expect = {ok, 15,[{{14,13,<<"13">>},13},
+ {{15,14,<<"14">>},14}]},
+
+ receive
+ {result, Result} ->
+ etap:is(Result, Expect, "stream OK.")
+ after 5000 ->
+ io:format("never got the change", [])
+ end.
+
+
+save_doc(Db, Id) ->
+ Doc = couch_mrview_test_util:doc(Id),
+ {ok, _Rev} = couch_db:update_doc(Db, Doc, []),
+ {ok, _} = couch_db:ensure_full_commit(Db),
+ couch_db:reopen(Db).
+
+run_query(Db, Opts) ->
+ Fun = fun
+ (stop, {LastSeq, Acc}) ->
+ {ok, LastSeq, Acc};
+ (heartbeat, Acc) ->
+ {ok, [heartbeat | Acc]};
+ (Event, Acc) ->
+ {ok, [Event | Acc]}
+ end,
+ couch_mrview:refresh(Db, <<"_design/bar">>),
+ {ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
+ <<"baz">>, Fun, [], Opts),
+ {ok, LastSeq, lists:reverse(R)}.
[05/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_index: add background indexing facility
This change add the possibility to trigger a view indexation in
background. The indexation can only work in background if at least one
process acquired it using the `couch_index_server:acquire_index/3`
function. If all the process that acquired it are down or released it
using `couch_index_server:release_indexer/3` then the background task is
stopped.
By default the background indexation will happen every 1s or when 200
docs has been saved in the database. These parameters can be changed
using the options `threshold` and `refresh_interval` in the couch_index
section.
To use it with couch_mrview a new option {refresh, true} has been added
to couch_mrview_changes:handle_changes Also the query parameter
refresh=true is passsed in t the HTTP changes API.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/5691328f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/5691328f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/5691328f
Branch: refs/heads/master
Commit: 5691328fbd219d70760ddc5d579d0874c9ff2272
Parents: dfe991f
Author: benoitc <be...@apache.org>
Authored: Sat Feb 8 19:55:40 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:33 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_changes.erl | 56 ++++++++++++++++++++++++++++-----------
test/10-index-changes.t | 17 +++++++++++-
2 files changed, 56 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/5691328f/src/couch_mrview_changes.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index a0e5281..c31624e 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -28,14 +28,16 @@
heartbeat,
timeout_acc=0,
notifier,
- stream}).
+ stream,
+ refresh}).
-type changes_stream() :: true | false | once.
-type changes_options() :: [{stream, changes_stream()} |
{since, integer()} |
{view_options, list()} |
{timeout, integer()} |
- {heartbeat, true | integer()}].
+ {heartbeat, true | integer()} |
+ {refresh, true | false}].
-export_type([changes_stream/0]).
-export_type([changes_options/0]).
@@ -47,6 +49,7 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
Since = proplists:get_value(since, Options, 0),
Stream = proplists:get_value(stream, Options, false),
ViewOptions = proplists:get_value(view_options, Options, []),
+ Refresh = proplists:get_value(refresh, Options, false),
State0 = #vst{dbname=DbName,
ddoc=DDocId,
@@ -56,20 +59,25 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
callback=Fun,
acc=Acc},
- case view_changes_since(State0) of
- {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
- case Stream of
- true ->
- start_loop(State#vst{stream=true}, Options);
- once when LastSeq =:= Since ->
- start_loop(State#vst{stream=once}, Options);
- _ ->
- Fun(stop, {LastSeq, Acc2})
- end;
- {stop, #vst{since=LastSeq, acc=Acc2}} ->
- Fun(stop, {LastSeq, Acc2});
- Error ->
- Error
+ maybe_acquire_indexer(Refresh, DbName, DDocId),
+ try
+ case view_changes_since(State0) of
+ {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
+ case Stream of
+ true ->
+ start_loop(State#vst{stream=true}, Options);
+ once when LastSeq =:= Since ->
+ start_loop(State#vst{stream=once}, Options);
+ _ ->
+ Fun(stop, {LastSeq, Acc2})
+ end;
+ {stop, #vst{since=LastSeq, acc=Acc2}} ->
+ Fun(stop, {LastSeq, Acc2});
+ Error ->
+ Error
+ end
+ after
+ maybe_release_indexer(Refresh, DbName, DDocId)
end.
start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
@@ -169,3 +177,19 @@ index_update_notifier(DbName, DDocId) ->
ok
end),
NotifierPid.
+
+%% acquire the background indexing task so it can eventually be started
+%% if the process close the background task will be automatically
+%% released.
+maybe_acquire_indexer(false, _, _) ->
+ ok;
+maybe_acquire_indexer(true, DbName, DDocId) ->
+ couch_index_server:acquire_indexer(couch_mrview_index, DbName,
+ DDocId).
+
+%% release the background indexing task so it can eventually be stopped
+maybe_release_indexer(false, _, _) ->
+ ok;
+maybe_release_indexer(true, DbName, DDocId) ->
+ couch_index_server:release_indexer(couch_mrview_index, DbName,
+ DDocId).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/5691328f/test/10-index-changes.t
----------------------------------------------------------------------
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index 627376f..f53e9ed 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -15,7 +15,7 @@
% the License.
main(_) ->
- etap:plan(6),
+ etap:plan(8),
case (catch test()) of
ok ->
etap:end_tests();
@@ -35,6 +35,7 @@ test() ->
test_stream_once_timeout(Db),
test_stream_once_heartbeat(Db),
test_stream(Db),
+ test_indexer(Db),
test_util:stop_couch(),
ok.
@@ -173,6 +174,20 @@ test_stream(Db) ->
end.
+test_indexer(Db) ->
+ Result = run_query(Db, [{since, 14}]),
+ Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
+ etap:is(Result, Expect, "refresh index by hand OK."),
+
+ {ok, Db1} = save_doc(Db, 15),
+ timer:sleep(1000),
+ Result1 = run_query(Db, [{since, 14}]),
+ Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
+ {{16,15,<<"15">>},15}]},
+ etap:is(Result1, Expect1, "changes indexed in background OK."),
+ ok.
+
+
save_doc(Db, Id) ->
Doc = couch_mrview_test_util:doc(Id),
{ok, _Rev} = couch_db:update_doc(Db, Doc, []),
[37/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Update expected return of couch_mrview_util:get_view
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/28e51f3f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/28e51f3f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/28e51f3f
Branch: refs/heads/master
Commit: 28e51f3f8cfefce6300243ec6ce1af35d4f454ea
Parents: e928480
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Oct 30 13:16:44 2014 -0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 19:28:42 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/28e51f3f/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index f6dec66..26d1cb2 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -215,8 +215,8 @@ trigger_update(Db, DDoc, UpdateSeq) ->
%% get informations on a view
get_view_info(Db, DDoc, VName) ->
- {ok, {_, View}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
- #mrargs{}),
+ {ok, {_, View, _}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+ #mrargs{}),
%% get the total number of rows
{ok, TotalRows} = couch_mrview_util:get_row_count(View),
[03/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
couch_mrview: fix purge when seq_indexed=true
This change makes tsure to also purge the log btree and related btrees
in views when seqs_indexed is set to true.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/deb8139b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/deb8139b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/deb8139b
Branch: refs/heads/master
Commit: deb8139b8ce94965abba6beb842b9579b569bd2a
Parents: 44c07c3
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 10:14:59 2014 +0100
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:38:32 2014 -0700
----------------------------------------------------------------------
src/couch_mrview.erl | 1 -
src/couch_mrview_updater.erl | 57 ++++++++++++++++++++++++++++-----------
2 files changed, 41 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/deb8139b/src/couch_mrview.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index afb6f1e..343b91a 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -150,7 +150,6 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
true -> View#mrview.key_byseq_btree;
_ -> View#mrview.seq_btree
end,
- io:format("opt list ~p~n", [OptList]),
AccOut = lists:foldl(fun(Opts, Acc0) ->
{ok, _R, A} = couch_mrview_util:fold_changes(
Btree, Fun, Acc0, Opts),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/deb8139b/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index eb06c92..27cf795 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -55,40 +55,66 @@ start_update(Partial, State, NumChanges) ->
purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
#mrst{
id_btree=IdBtree,
+ log_btree=LogBtree,
views=Views
} = State,
Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
- {ok, Lookups, IdBtree2} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+ {ok, Lookups, LLookups, LogBtree2, IdBtree2} = case LogBtree of
+ nil ->
+ {ok, L, Bt} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+ {ok, L, [], nil, Bt};
+ _ ->
+ {ok, L, Bt} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+ {ok, LL, LBt} = couch_btree:query_modify(LogBtree, Ids, [], Ids),
+ {ok, L, LL, LBt, Bt}
+ end,
MakeDictFun = fun
({ok, {DocId, ViewNumRowKeys}}, DictAcc) ->
- FoldFun = fun({ViewNum, RowKey}, DictAcc2) ->
- dict:append(ViewNum, {RowKey, DocId}, DictAcc2)
+ FoldFun = fun
+ ({ViewNum, {Key, Seq, _Op}}, DictAcc2) ->
+ dict:append(ViewNum, {Key, Seq, DocId}, DictAcc2);
+ ({ViewNum, RowKey}, DictAcc2) ->
+ dict:append(ViewNum, {RowKey, DocId}, DictAcc2)
end,
lists:foldl(FoldFun, DictAcc, ViewNumRowKeys);
({not_found, _}, DictAcc) ->
DictAcc
end,
KeysToRemove = lists:foldl(MakeDictFun, dict:new(), Lookups),
+ SeqsToRemove = lists:foldl(MakeDictFun, dict:new(), LLookups),
+
+ RemKeysFun = fun(#mrview{id_num=ViewId}=View) ->
+ ToRem = couch_util:dict_find(ViewId, KeysToRemove, []),
+ {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, [], ToRem),
+ NewPurgeSeq = case VBtree2 =/= View#mrview.btree of
+ true -> PurgeSeq;
+ _ -> View#mrview.purge_seq
+ end,
+ {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
+ true ->
+ SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+ SKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+ KSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
+ {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
+ [], SKs),
+ {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
+ [], KSs),
+ {SBt, KSbt};
+ _ -> {nil, nil}
+ end,
+ View#mrview{btree=VBtree2,
+ seq_btree=SeqBtree2,
+ key_byseq_btree=KeyBySeqBtree2,
+ purge_seq=NewPurgeSeq}
- RemKeysFun = fun(#mrview{id_num=Num, btree=Btree}=View) ->
- case dict:find(Num, KeysToRemove) of
- {ok, RemKeys} ->
- {ok, Btree2} = couch_btree:add_remove(Btree, [], RemKeys),
- NewPurgeSeq = case Btree2 /= Btree of
- true -> PurgeSeq;
- _ -> View#mrview.purge_seq
- end,
- View#mrview{btree=Btree2, purge_seq=NewPurgeSeq};
- error ->
- View
- end
end,
Views2 = lists:map(RemKeysFun, Views),
{ok, State#mrst{
id_btree=IdBtree2,
+ log_btree=LogBtree2,
views=Views2,
purge_seq=PurgeSeq
}}.
@@ -345,7 +371,6 @@ update_log(Btree, Log, UpdatedSeq, _) ->
end, KeysAcc, DIKeys),
{[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
- io:format("updated ~p~n", [Updated]),
RemValue = {[{<<"_removed">>, true}]},
{Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
{Log2, AddAcc2, DelAcc2}) ->
[24/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Posted by bb...@apache.org.
Add rev to view changes respose
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/b3fef4cb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/b3fef4cb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/b3fef4cb
Branch: refs/heads/master
Commit: b3fef4cbfcd02178248cedcef5057604ba2d7130
Parents: 24db15f
Author: Benjamin Bastian <be...@gmail.com>
Authored: Mon Aug 25 16:49:07 2014 +0700
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:40:29 2014 -0700
----------------------------------------------------------------------
src/couch_mrview_updater.erl | 40 ++++++++++++++++++++-------------------
src/couch_mrview_util.erl | 2 +-
2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/b3fef4cb/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 16645aa..3fe1dd6 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -124,11 +124,13 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
couch_work_queue:queue(State#mrst.doc_queue, lists:reverse(Acc)),
process_doc(Doc, Seq, State#mrst{doc_acc=[]});
process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
- {ok, State#mrst{doc_acc=[{nil, Seq, nil} | Acc]}};
-process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
- {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
+ {ok, State#mrst{doc_acc=[{nil, Seq, nil, nil} | Acc]}};
+process_doc(#doc{id=Id, deleted=true}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
+ {RevPos, [Rev | _]} = Doc#doc.revs,
+ {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, deleted} | Acc]}};
process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
- {ok, State#mrst{doc_acc=[{Id, Seq, Doc} | Acc]}}.
+ {RevPos, [Rev | _]} = Doc#doc.revs,
+ {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, Doc} | Acc]}}.
finish_update(#mrst{doc_acc=Acc}=State) ->
@@ -164,15 +166,15 @@ map_docs(Parent, State0) ->
end,
QServer = State1#mrst.qserver,
DocFun = fun
- ({nil, Seq, _}, {SeqAcc, Results}) ->
+ ({nil, Seq, _, _}, {SeqAcc, Results}) ->
{erlang:max(Seq, SeqAcc), Results};
- ({Id, Seq, deleted}, {SeqAcc, Results}) ->
- {erlang:max(Seq, SeqAcc), [{Id, Seq, []} | Results]};
- ({Id, Seq, Doc}, {SeqAcc, Results}) ->
+ ({Id, Seq, Rev, deleted}, {SeqAcc, Results}) ->
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, []} | Results]};
+ ({Id, Seq, Rev, Doc}, {SeqAcc, Results}) ->
couch_stats:increment_counter([couchdb, mrview, map_docs],
1),
{ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
- {erlang:max(Seq, SeqAcc), [{Id, Seq, Res} | Results]}
+ {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, Res} | Results]}
end,
FoldFun = fun(Docs, Acc) ->
update_task(length(Docs)),
@@ -250,27 +252,27 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
Log1).
-merge_results({DocId, _Seq, []}, ViewKVs, DocIdKeys, Log) ->
- {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, [], Log)};
-merge_results({DocId, Seq, RawResults}, ViewKVs, DocIdKeys, Log) ->
+merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
+ {ViewKVs, [{DocId, Rev, []} | DocIdKeys], dict:store(DocId, [], Log)};
+merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
JsonResults = couch_query_servers:raw_to_ejson(RawResults),
Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
- {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Results, ViewKVs, [],
+ {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [],
[], Log),
{ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
-insert_results(DocId, _Seq, [], [], ViewKVs, ViewIdKeys, Log) ->
+insert_results(DocId, _Seq, _Rev, [], [], ViewKVs, ViewIdKeys, Log) ->
{lists:reverse(ViewKVs), {DocId, ViewIdKeys}, Log};
-insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
+insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
VIdKeys, Log) ->
CombineDupesFun = fun
({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys, Log2}) ->
{[{Key, {dups, [Val | Vals]}} | Rest], IdKeys, Log2};
({Key, Val1}, {[{Key, Val2} | Rest], IdKeys, Log2}) ->
{[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
- ({Key, _}=KV, {Rest, IdKeys, Log2}) ->
- {[KV | Rest], [{Id, Key} | IdKeys],
+ ({Key, Value}, {Rest, IdKeys, Log2}) ->
+ {[{Key, Value} | Rest], [{Id, Key} | IdKeys],
dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
end,
InitAcc = {[], VIdKeys, Log},
@@ -278,8 +280,8 @@ insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
{Duped, VIdKeys0, Log1} = lists:foldl(CombineDupesFun, InitAcc,
lists:sort(KVs)),
FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs,
- FinalSKVs = [{{Seq, Key}, {DocId, Val}} || {Key, Val} <- Duped] ++ SKVs,
- insert_results(DocId, Seq, RKVs, RVKVs,
+ FinalSKVs = [{{Seq, Key}, {DocId, Val, Rev}} || {Key, Val} <- Duped] ++ SKVs,
+ insert_results(DocId, Seq, Rev, RKVs, RVKVs,
[{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/b3fef4cb/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index f5fd66c..734ad26 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -854,7 +854,7 @@ mrverror(Mesg) ->
to_key_seq(L) ->
- [{{[Key, Seq], DocId}, Val} || {{Seq, Key}, {DocId, Val}} <- L].
+ [{{[Key, Seq], DocId}, {Val, Rev}} || {{Seq, Key}, {DocId, Val, Rev}} <- L].
%% Updates 1.2.x or earlier view files to 1.3.x or later view files
%% transparently, the first time the 1.2.x view file is opened by