You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/02/13 00:29:03 UTC
[08/25] couch-mrview commit: updated refs/heads/1994-merge-rcouch to
7775266
couch_mrview: fix compaction with seqs_indexed=true
This change smakes sure to compact the view indexes when using the view
changes.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/3d27c6be
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/3d27c6be
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/3d27c6be
Branch: refs/heads/1994-merge-rcouch
Commit: 3d27c6be52aada0bfd16b484e820e6df49626ca7
Parents: 188b30f
Author: benoitc <be...@apache.org>
Authored: Mon Jan 27 15:29:10 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Feb 12 17:27:39 2014 -0600
----------------------------------------------------------------------
src/couch_mrview_compactor.erl | 84 +++++++++++++++++++++++++++++++++----
src/couch_mrview_util.erl | 12 +++++-
2 files changed, 88 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/3d27c6be/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 4ab8dd1..a7e594f 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -40,6 +40,8 @@ compact(State) ->
sig=Sig,
update_seq=Seq,
id_btree=IdBtree,
+ log_btree=LogBtree,
+ seq_indexed=SeqIndexed,
views=Views
} = State,
@@ -56,15 +58,28 @@ compact(State) ->
#mrst{
id_btree = EmptyIdBtree,
+ log_btree = EmptyLogBtree,
views = EmptyViews
} = EmptyState,
+ TotalChanges0 = case SeqIndexed of
+ true -> NumDocIds * 2;
+ _ -> NumDocIds
+ end,
+
TotalChanges = lists:foldl(
fun(View, Acc) ->
{ok, Kvs} = couch_mrview_util:get_row_count(View),
- Acc + Kvs
+ case SeqIndexed of
+ true ->
+ {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
+ Acc + Kvs + SKvs * 2;
+ false ->
+ Acc + Kvs
+ end
end,
- NumDocIds, Views),
+ TotalChanges0, Views),
+
couch_task_status:add_task([
{type, view_compaction},
{database, DbName},
@@ -106,13 +121,25 @@ compact(State) ->
{ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+
+ {NewLogBtree, FinalAcc3} = case SeqIndexed of
+ true ->
+ compact_log(LogBtree, BufferSize,
+ FinalAcc2#acc{kvs=[],
+ kvs_size=0,
+ btree=EmptyLogBtree});
+ _ ->
+ {nil, FinalAcc2}
+ end,
+
{NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
compact_view(View, EmptyView, BufferSize, Acc)
- end, FinalAcc2, lists:zip(Views, EmptyViews)),
+ end, FinalAcc3, lists:zip(Views, EmptyViews)),
unlink(EmptyState#mrst.fd),
{ok, EmptyState#mrst{
id_btree=NewIdBtree,
+ log_btree=NewLogBtree,
views=NewViews,
update_seq=Seq
}}.
@@ -129,9 +156,53 @@ recompact(State) ->
{ok, State2}
end.
+compact_log(LogBtree, BufferSize, Acc0) ->
+ FoldFun = fun({DocId, _} = KV, Acc) ->
+ #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+ KvsSize2 = KvsSize + ?term_size(KV),
+ case KvsSize2 >= BufferSize of
+ true ->
+ {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+ Acc2 = update_task(Acc, 1 + length(Kvs)),
+ {ok, Acc2#acc{
+ btree = Bt2, kvs = [], kvs_size = 0}};
+ _ ->
+ {ok, Acc#acc{
+ kvs = [KV | Kvs], kvs_size = KvsSize2}}
+ end
+ end,
+
+ {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0),
+ #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+ {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+ FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+ {NewLogBtree, FinalAcc2}.
%% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
compact_view(View, EmptyView, BufferSize, Acc0) ->
+
+ {NewBt, Acc1} = compact_view_btree(View#mrview.btree,
+ EmptyView#mrview.btree,
+ BufferSize, Acc0),
+
+ %% are we indexing changes by sequences?
+ {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+ true ->
+ {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
+ EmptyView#mrview.seq_btree,
+ BufferSize, Acc1),
+ {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
+ EmptyView#mrview.key_byseq_btree,
+ BufferSize, Acc2),
+ {SBt, KSBt, Acc3};
+ _ ->
+ {nil, nil, Acc1}
+ end,
+ {EmptyView#mrview{btree=NewBt,
+ seq_btree=NewSeqBt,
+ key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
+
+compact_view_btree(Btree, EmptyBtree, BufferSize, Acc0) ->
Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
KvsSize2 = KvsSize + ?term_size(KV),
if KvsSize2 >= BufferSize ->
@@ -143,13 +214,12 @@ compact_view(View, EmptyView, BufferSize, Acc0) ->
end
end,
- InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
- {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
+ InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree},
+ {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc),
#acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
{ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
- {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
-
+ {NewBt, FinalAcc2}.
update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) ->
Changes2 = Changes + ChangesInc,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/3d27c6be/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 8cb4fae..7c42de8 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -18,6 +18,7 @@
-export([index_file/2, compaction_file/2, open_file/1]).
-export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
-export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_view_changes_count/1]).
-export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
-export([fold/4, fold_reduce/4]).
-export([temp_view_to_ddoc/1]).
@@ -330,6 +331,12 @@ reduce_to_count(Reductions) ->
{Count, _} = couch_btree:final_reduce(Reduce, Reductions),
Count.
+%% @doc get all changes for a view
+get_view_changes_count(#mrview{seq_btree=Btree}) ->
+ couch_btree:fold_reduce(
+ Btree, fun(_SeqStart, PartialReds, 0) ->
+ {ok, couch_btree:final_reduce(Btree, PartialReds)}
+ end,0, []).
fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
WrapperFun = fun(KV, Reds, Acc2) ->
@@ -625,10 +632,13 @@ reset_state(State) ->
State#mrst{
fd=nil,
qserver=nil,
+ seq_indexed=State#mrst.seq_indexed,
update_seq=0,
id_btree=nil,
log_btree=nil,
- views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+ views=[View#mrview{btree=nil, seq_btree=nil,
+ key_byseq_btree=nil,
+ seq_indexed=View#mrview.seq_indexed}
|| View <- State#mrst.views]
}.