[couchdb] branch ebtree-view-tracing created (now 7130276)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch ebtree-view-tracing
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


      at 7130276  Measure view build stats

This branch includes the following new commits:

     new 7130276  Measure view build stats

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[couchdb] 01/01: Measure view build stats

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch ebtree-view-tracing
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 713027646750147c1f84b43e0d33a8fa2e5ecfc5
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Aug 13 11:58:45 2020 -0500

    Measure view build stats
---
 src/couch_views/src/couch_views_indexer.erl | 69 ++++++++++++++++++++++++++---
 1 file changed, 64 insertions(+), 5 deletions(-)

diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index 858a988..0d2554b 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -178,12 +178,15 @@ add_error(Error, Reason, Data) ->
 update(#{} = Db, Mrst0, State0) ->
     Limit = couch_views_batch:start(Mrst0),
     {Mrst1, State1} = try
-        do_update(Db, Mrst0, State0#{limit => Limit})
+        time_span(do_update, fun() ->
+            do_update(Db, Mrst0, State0#{limit => Limit})
+        end)
     catch
         error:{erlfdb_error, Error} when ?IS_RECOVERABLE_ERROR(Error) ->
             couch_views_batch:failure(Mrst0),
             update(Db, Mrst0, State0)
     end,
+    stat_dump(),
     case State1 of
         finished ->
             couch_eval:release_map_context(Mrst1#mrst.qserver);
@@ -205,7 +208,9 @@ do_update(Db, Mrst0, State0) ->
         State1 = get_update_start_state(TxDb, Mrst0, State0),
         Mrst1 = couch_views_fdb:set_trees(TxDb, Mrst0),
 
-        {ok, State2} = fold_changes(State1),
+        {ok, State2} = time_span(fold_changes, fun() ->
+            fold_changes(State1)
+        end),
 
         #{
             count := Count,
@@ -217,10 +222,19 @@ do_update(Db, Mrst0, State0) ->
             design_opts := DesignOpts
         } = State2,
 
-        DocAcc1 = fetch_docs(TxDb, DesignOpts, DocAcc),
+        stat_incr(changes_read, length(DocAcc)),
 
-        {Mrst2, MappedDocs} = map_docs(Mrst0, DocAcc1),
-        TotalKVs = write_docs(TxDb, Mrst1, MappedDocs, State2),
+        DocAcc1 = time_span(fetch_docs, fun() ->
+            fetch_docs(TxDb, DesignOpts, DocAcc)
+        end),
+
+        {Mrst2, MappedDocs} = time_span(map_docs, fun() ->
+            map_docs(Mrst1, DocAcc1)
+        end),
+
+        TotalKVs = time_span(write_docs, fun() ->
+            write_docs(TxDb, Mrst2, MappedDocs, State2)
+        end),
 
         ChangesDone = ChangesDone0 + length(DocAcc),
 
@@ -599,6 +613,51 @@ fail_job(Job, Data, Error, Reason) ->
     exit(normal).
 
 
+time_span(Id, Fun) ->
+    Start = erlang:system_time(microsecond),
+    try
+        Fun()
+    after
+        Diff = erlang:system_time(microsecond) - Start,
+        stat_store(Id, Diff / 1000000)
+    end.
+
+
+stat_incr(Id, Count) ->
+    case get('$view_stats') of
+        #{Id := OldCount} ->
+            stat_store(Id, OldCount + Count);
+        _ ->
+            stat_store(Id, Count)
+    end.
+
+
+stat_store(Id, Value) ->
+    NewStats = case get('$view_stats') of
+        #{} = Stats ->
+            maps:put(Id, Value, Stats);
+        undefined ->
+            #{Id => Value}
+    end,
+    put('$view_stats', NewStats).
+
+
+stat_dump() ->
+    case get('$view_stats') of
+        #{} = Stats ->
+            KVs = lists:sort(maps:to_list(Stats)),
+            Strs = lists:foldl(fun({Id, Value}, Acc) ->
+                Str = io_lib:format("~s:~w", [Id, Value]),
+                [Str | Acc]
+            end, [], KVs),
+            Msg = "XKCD VIEW STATS: " ++ string:join(lists:reverse(Strs),  " "),
+            couch_log:error(Msg, []),
+            put('$view_stats', #{});
+        _ ->
+            ok
+    end.
+
+
 retry_limit() ->
     config:get_integer("couch_views", "retry_limit", 3).