You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/01/17 23:09:49 UTC

[04/49] git commit: Improved _active_tasks API

Improved _active_tasks API

Tasks are now free to set any properties they wish (as an
Erlang proplist). Different tasks can have different properties
and the status string doesn't exist anymore - instead client
applications can build it using more granular properties from
_active_tasks. Some of these properties are:

1) "progress" (an integer percentage, for all tasks)
2) "database" (for compactions and indexer tasks)
3) "design_document" (for indexer and view compaction tasks)
4) "source" and "target" (for replications)
5) "docs_read", "docs_written", "doc_write_failures",
   "missing_revs_found", "missing_revs_checked", "source_seq",
   "checkpointed_source_seq" and "continuous" for replications



git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1171888 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/f3ccaa16
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/f3ccaa16
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/f3ccaa16

Branch: refs/heads/import
Commit: f3ccaa16f266fd0b4036a3bc8eec1a87cb0bdf4f
Parents: 2beb107
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Sat Sep 17 03:31:12 2011 +0000
Committer: Filipe David Borba Manana <fd...@apache.org>
Committed: Sat Sep 17 03:31:12 2011 +0000

----------------------------------------------------------------------
 src/couch_mrview_compactor.erl | 84 ++++++++++++++++++++++++-------------
 src/couch_mrview_index.erl     |  6 +--
 src/couch_mrview_updater.erl   | 31 ++++++++++++--
 3 files changed, 86 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index b67de01..ef3f158 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -17,6 +17,15 @@
 
 -export([compact/2, swap_compacted/2]).
 
+-record(acc, {
+   btree = nil,
+   last_id = nil,
+   kvs = [],
+   kvs_size = 0,
+   changes = 0,
+   total_changes
+}).
+
 
 compact(State, Opts) ->
     case lists:member(recompact, Opts) of
@@ -46,15 +55,26 @@ compact(State) ->
     } = EmptyState,
 
     {ok, Count} = couch_btree:full_reduce(IdBtree),
-    TaskName = <<DbName/binary, ":", IdxName/binary>>,
-    couch_task_status:add_task(<<"View Group Compaction">>, TaskName, <<"">>),
+    TotalChanges = lists:foldl(
+        fun(View, Acc) ->
+            {ok, Kvs} = couch_mrview_util:get_row_count(View),
+            Acc + Kvs
+        end,
+        Count, Views),
+    couch_task_status:add_task([
+        {type, view_compaction},
+        {database, DbName},
+        {design_document, IdxName},
+        {progress, 0}
+    ]),
 
     BufferSize0 = couch_config:get(
         "view_compaction", "keyvalue_buffer_size", "2097152"
     ),
     BufferSize = list_to_integer(BufferSize0),
 
-    FoldFun = fun({DocId, _} = KV, {Bt, Acc, AccSize, Copied, LastId}) ->
+    FoldFun = fun({DocId, _} = KV, Acc) ->
+        #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize, last_id = LastId} = Acc,
         if DocId =:= LastId ->
             % COUCHDB-999 regression test
             ?LOG_ERROR("Duplicate docid `~s` detected in view group `~s`"
@@ -63,26 +83,28 @@ compact(State) ->
             ),
             exit({view_duplicate_id, DocId});
         true -> ok end,
-        AccSize2 = AccSize + ?term_size(KV),
-        case AccSize2 >= BufferSize of
+        KvsSize2 = KvsSize + ?term_size(KV),
+        case KvsSize2 >= BufferSize of
             true ->
-                {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV|Acc])),
-                couch_task_status:update("Copied ~p of ~p Ids (~p%)",
-                    [Copied, Count, (Copied * 100) div Count]),
-                {ok, {Bt2, [], 0, Copied+1+length(Acc), DocId}};
+                {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+                Acc2 = update_task(Acc, 1 + length(Kvs)),
+                {ok, Acc2#acc{
+                    btree = Bt2, kvs = [], kvs_size = 0, last_id = DocId}};
             _ ->
-                {ok, {Bt, [KV | Acc], AccSize2, Copied, DocId}}
+                {ok, Acc#acc{
+                    kvs = [KV | Kvs], kvs_size = KvsSize2, last_id = DocId}}
         end
     end,
 
-    InitAcc = {EmptyIdBtree, [], 0, 0, nil},
+    InitAcc = #acc{total_changes = TotalChanges, btree = EmptyIdBtree},
     {ok, _, FinalAcc} = couch_btree:foldl(IdBtree, FoldFun, InitAcc),
-    {Bt3, Uncopied, _, _, _} = FinalAcc,
+    #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
     {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+    FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
 
-    NewViews = lists:map(fun({View, EmptyView}) ->
-        compact_view(View, EmptyView, BufferSize)
-    end, lists:zip(Views, EmptyViews)),
+    {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
+        compact_view(View, EmptyView, BufferSize, Acc)
+    end, FinalAcc2, lists:zip(Views, EmptyViews)),
 
     unlink(EmptyState#mrst.fd),
     {ok, EmptyState#mrst{
@@ -109,27 +131,31 @@ recompact(State) ->
     end.
 
 
-%% @spec compact_view(View, EmptyView, Retry) -> CompactView
-compact_view(View, EmptyView, BufferSize) ->
-    {ok, Count} = couch_mrview_util:get_row_count(View),
-    Fun = fun(KV, {Bt, Acc, AccSize, Copied}) ->
-        AccSize2 = AccSize + ?term_size(KV),
-        if AccSize2 >= BufferSize ->
-            {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV|Acc])),
-            couch_task_status:update("View #~p: copied ~p of ~p KVs (~p%)",
-                [View#mrview.id_num, Copied, Count, (Copied*100) div Count]),
-            {ok, {Bt2, [], 0, Copied + 1 + length(Acc)}};
+%% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
+compact_view(View, EmptyView, BufferSize, Acc0) ->
+    Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
+        KvsSize2 = KvsSize + ?term_size(KV),
+        if KvsSize2 >= BufferSize ->
+            {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+            Acc2 = update_task(Acc, 1 + length(Kvs)),
+            {ok, Acc2#acc{btree = Bt2, kvs = [], kvs_size = 0}};
         true ->
-            {ok, {Bt, [KV|Acc], AccSize2, Copied}}
+            {ok, Acc#acc{kvs = [KV | Kvs], kvs_size = KvsSize2}}
         end
     end,
 
-    InitAcc = {EmptyView#mrview.btree, [], 0, 0},
+    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
     {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
-    {Bt3, Uncopied, _, _} = FinalAcc,
+    #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
     {ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+    FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+    {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
+
 
-    EmptyView#mrview{btree=NewBt}.
+update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) ->
+    Changes2 = Changes + ChangesInc,
+    couch_task_status:update([{progress, (Changes2 * 100) div Total}]),
+    Acc#acc{changes = Changes2}.
 
 
 swap_compacted(OldState, NewState) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 7fbe06f..d2d700d 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -15,7 +15,7 @@
 
 -export([get/2]).
 -export([init/2, open/2, close/1, reset/1, delete/1]).
--export([start_update/2, purge/4, process_doc/3, finish_update/1, commit/1]).
+-export([start_update/3, purge/4, process_doc/3, finish_update/1, commit/1]).
 -export([compact/2, swap_compacted/2]).
 
 
@@ -106,8 +106,8 @@ reset(State) ->
     end).
 
 
-start_update(PartialDest, State) ->
-    couch_mrview_updater:start_update(PartialDest, State).
+start_update(PartialDest, State, NumChanges) ->
+    couch_mrview_updater:start_update(PartialDest, State, NumChanges).
 
 
 purge(Db, PurgeSeq, PurgedIdRevs, State) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f3ccaa16/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index e95290b..9604ea9 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -12,13 +12,13 @@
 
 -module(couch_mrview_updater).
 
--export([start_update/2, purge/4, process_doc/3, finish_update/1]).
+-export([start_update/3, purge/4, process_doc/3, finish_update/1]).
 
 -include("couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
 
-start_update(Partial, State) ->
+start_update(Partial, State, NumChanges) ->
     QueueOpts = [{max_size, 100000}, {max_items, 500}],
     {ok, DocQueue} = couch_work_queue:new(QueueOpts),
     {ok, WriteQueue} = couch_work_queue:new(QueueOpts),
@@ -32,7 +32,18 @@ start_update(Partial, State) ->
     },
 
     Self = self(),
-    MapFun = fun() -> map_docs(Self, InitState) end,
+    MapFun = fun() ->
+        couch_task_status:add_task([
+            {type, indexer},
+            {database, State#mrst.db_name},
+            {design_document, State#mrst.idx_name},
+            {progress, 0},
+            {changes_done, 0},
+            {total_changes, NumChanges}
+        ]),
+        couch_task_status:set_update_frequency(500),
+        map_docs(Self, InitState)
+    end,
     WriteFun = fun() -> write_results(Self, InitState) end,
 
     spawn_link(MapFun),
@@ -136,6 +147,7 @@ map_docs(Parent, State0) ->
                     {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
             end,
             FoldFun = fun(Docs, Acc) ->
+                update_task(length(Docs)),
                 lists:foldl(DocFun, Acc, Docs)
             end,
             Results = lists:foldl(FoldFun, {0, []}, Dequeued),
@@ -255,3 +267,16 @@ send_partial(Pid, State) when is_pid(Pid) ->
     gen_server:cast(Pid, {new_state, State});
 send_partial(_, _) ->
     ok.
+
+
+update_task(NumChanges) ->
+    [Changes, Total] = couch_task_status:get([changes_done, total_changes]),
+    Changes2 = Changes + NumChanges,
+    Progress = case Total of
+        0 ->
+            % updater restart after compaction finishes
+            0;
+        _ ->
+            (Changes2 * 100) div Total
+    end,
+    couch_task_status:update([{progress, Progress}, {changes_done, Changes2}]).