You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/01/17 23:09:49 UTC
[04/49] git commit: Improved _active_tasks API
Improved _active_tasks API
Tasks are now free to set any properties they wish (as an
Erlang proplist). Different tasks can have different properties
and the status string doesn't exist anymore - instead client
applications can build it using more granular properties from
_active_tasks. Some of these properties are:
1) "progress" (an integer percentage, for all tasks)
2) "database" (for compactions and indexer tasks)
3) "design_document" (for indexer and view compaction tasks)
4) "source" and "target" (for replications)
5) "docs_read", "docs_written", "doc_write_failures",
"missing_revs_found", "missing_revs_checked", "source_seq",
"checkpointed_source_seq" and "continuous" for replications
git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1171888 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/f3ccaa16
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/f3ccaa16
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/f3ccaa16
Branch: refs/heads/import
Commit: f3ccaa16f266fd0b4036a3bc8eec1a87cb0bdf4f
Parents: 2beb107
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Sat Sep 17 03:31:12 2011 +0000
Committer: Filipe David Borba Manana <fd...@apache.org>
Committed: Sat Sep 17 03:31:12 2011 +0000
----------------------------------------------------------------------
src/couch_mrview_compactor.erl | 84 ++++++++++++++++++++++++-------------
src/couch_mrview_index.erl | 6 +--
src/couch_mrview_updater.erl | 31 ++++++++++++--
3 files changed, 86 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index b67de01..ef3f158 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -17,6 +17,15 @@
-export([compact/2, swap_compacted/2]).
+-record(acc, {
+ btree = nil,
+ last_id = nil,
+ kvs = [],
+ kvs_size = 0,
+ changes = 0,
+ total_changes
+}).
+
compact(State, Opts) ->
case lists:member(recompact, Opts) of
@@ -46,15 +55,26 @@ compact(State) ->
} = EmptyState,
{ok, Count} = couch_btree:full_reduce(IdBtree),
- TaskName = <<DbName/binary, ":", IdxName/binary>>,
- couch_task_status:add_task(<<"View Group Compaction">>, TaskName, <<"">>),
+ TotalChanges = lists:foldl(
+ fun(View, Acc) ->
+ {ok, Kvs} = couch_mrview_util:get_row_count(View),
+ Acc + Kvs
+ end,
+ Count, Views),
+ couch_task_status:add_task([
+ {type, view_compaction},
+ {database, DbName},
+ {design_document, IdxName},
+ {progress, 0}
+ ]),
BufferSize0 = couch_config:get(
"view_compaction", "keyvalue_buffer_size", "2097152"
),
BufferSize = list_to_integer(BufferSize0),
- FoldFun = fun({DocId, _} = KV, {Bt, Acc, AccSize, Copied, LastId}) ->
+ FoldFun = fun({DocId, _} = KV, Acc) ->
+ #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize, last_id = LastId} = Acc,
if DocId =:= LastId ->
% COUCHDB-999 regression test
?LOG_ERROR("Duplicate docid `~s` detected in view group `~s`"
@@ -63,26 +83,28 @@ compact(State) ->
),
exit({view_duplicate_id, DocId});
true -> ok end,
- AccSize2 = AccSize + ?term_size(KV),
- case AccSize2 >= BufferSize of
+ KvsSize2 = KvsSize + ?term_size(KV),
+ case KvsSize2 >= BufferSize of
true ->
- {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV|Acc])),
- couch_task_status:update("Copied ~p of ~p Ids (~p%)",
- [Copied, Count, (Copied * 100) div Count]),
- {ok, {Bt2, [], 0, Copied+1+length(Acc), DocId}};
+ {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+ Acc2 = update_task(Acc, 1 + length(Kvs)),
+ {ok, Acc2#acc{
+ btree = Bt2, kvs = [], kvs_size = 0, last_id = DocId}};
_ ->
- {ok, {Bt, [KV | Acc], AccSize2, Copied, DocId}}
+ {ok, Acc#acc{
+ kvs = [KV | Kvs], kvs_size = KvsSize2, last_id = DocId}}
end
end,
- InitAcc = {EmptyIdBtree, [], 0, 0, nil},
+ InitAcc = #acc{total_changes = TotalChanges, btree = EmptyIdBtree},
{ok, _, FinalAcc} = couch_btree:foldl(IdBtree, FoldFun, InitAcc),
- {Bt3, Uncopied, _, _, _} = FinalAcc,
+ #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
{ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+ FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
- NewViews = lists:map(fun({View, EmptyView}) ->
- compact_view(View, EmptyView, BufferSize)
- end, lists:zip(Views, EmptyViews)),
+ {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
+ compact_view(View, EmptyView, BufferSize, Acc)
+ end, FinalAcc2, lists:zip(Views, EmptyViews)),
unlink(EmptyState#mrst.fd),
{ok, EmptyState#mrst{
@@ -109,27 +131,31 @@ recompact(State) ->
end.
-%% @spec compact_view(View, EmptyView, Retry) -> CompactView
-compact_view(View, EmptyView, BufferSize) ->
- {ok, Count} = couch_mrview_util:get_row_count(View),
- Fun = fun(KV, {Bt, Acc, AccSize, Copied}) ->
- AccSize2 = AccSize + ?term_size(KV),
- if AccSize2 >= BufferSize ->
- {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV|Acc])),
- couch_task_status:update("View #~p: copied ~p of ~p KVs (~p%)",
- [View#mrview.id_num, Copied, Count, (Copied*100) div Count]),
- {ok, {Bt2, [], 0, Copied + 1 + length(Acc)}};
+%% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
+compact_view(View, EmptyView, BufferSize, Acc0) ->
+ Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
+ KvsSize2 = KvsSize + ?term_size(KV),
+ if KvsSize2 >= BufferSize ->
+ {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+ Acc2 = update_task(Acc, 1 + length(Kvs)),
+ {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0}};
true ->
- {ok, {Bt, [KV|Acc], AccSize2, Copied}}
+ {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2}}
end
end,
- InitAcc = {EmptyView#mrview.btree, [], 0, 0},
+ InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
{ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
- {Bt3, Uncopied, _, _} = FinalAcc,
+ #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
{ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+ FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+ {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
+
- EmptyView#mrview{btree=NewBt}.
+update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) ->
+ Changes2 = Changes + ChangesInc,
+ couch_task_status:update([{progress, (Changes2 * 100) div Total}]),
+ Acc#acc{changes = Changes2}.
swap_compacted(OldState, NewState) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 7fbe06f..d2d700d 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -15,7 +15,7 @@
-export([get/2]).
-export([init/2, open/2, close/1, reset/1, delete/1]).
--export([start_update/2, purge/4, process_doc/3, finish_update/1, commit/1]).
+-export([start_update/3, purge/4, process_doc/3, finish_update/1, commit/1]).
-export([compact/2, swap_compacted/2]).
@@ -106,8 +106,8 @@ reset(State) ->
end).
-start_update(PartialDest, State) ->
- couch_mrview_updater:start_update(PartialDest, State).
+start_update(PartialDest, State, NumChanges) ->
+ couch_mrview_updater:start_update(PartialDest, State, NumChanges).
purge(Db, PurgeSeq, PurgedIdRevs, State) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index e95290b..9604ea9 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -12,13 +12,13 @@
-module(couch_mrview_updater).
--export([start_update/2, purge/4, process_doc/3, finish_update/1]).
+-export([start_update/3, purge/4, process_doc/3, finish_update/1]).
-include("couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
-start_update(Partial, State) ->
+start_update(Partial, State, NumChanges) ->
QueueOpts = [{max_size, 100000}, {max_items, 500}],
{ok, DocQueue} = couch_work_queue:new(QueueOpts),
{ok, WriteQueue} = couch_work_queue:new(QueueOpts),
@@ -32,7 +32,18 @@ start_update(Partial, State) ->
},
Self = self(),
- MapFun = fun() -> map_docs(Self, InitState) end,
+ MapFun = fun() ->
+ couch_task_status:add_task([
+ {type, indexer},
+ {database, State#mrst.db_name},
+ {design_document, State#mrst.idx_name},
+ {progress, 0},
+ {changes_done, 0},
+ {total_changes, NumChanges}
+ ]),
+ couch_task_status:set_update_frequency(500),
+ map_docs(Self, InitState)
+ end,
WriteFun = fun() -> write_results(Self, InitState) end,
spawn_link(MapFun),
@@ -136,6 +147,7 @@ map_docs(Parent, State0) ->
{erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
end,
FoldFun = fun(Docs, Acc) ->
+ update_task(length(Docs)),
lists:foldl(DocFun, Acc, Docs)
end,
Results = lists:foldl(FoldFun, {0, []}, Dequeued),
@@ -255,3 +267,16 @@ send_partial(Pid, State) when is_pid(Pid) ->
gen_server:cast(Pid, {new_state, State});
send_partial(_, _) ->
ok.
+
+
+update_task(NumChanges) ->
+ [Changes, Total] = couch_task_status:get([changes_done, total_changes]),
+ Changes2 = Changes + NumChanges,
+ Progress = case Total of
+ 0 ->
+ % updater restart after compaction finishes
+ 0;
+ _ ->
+ (Changes2 * 100) div Total
+ end,
+ couch_task_status:update([{progress, Progress}, {changes_done, Changes2}]).