You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by jx...@apache.org on 2018/03/28 18:05:27 UTC

[incubator-mxnet] branch master updated: Refreshed branch aggregate (#10228)

This is an automated email from the ASF dual-hosted git repository.

jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new e5f6238  Refreshed branch aggregate (#10228)
e5f6238 is described below

commit e5f6238a2b213e70ba0ca7acc5bcf5c643502561
Author: Chris Olivier <cj...@gmail.com>
AuthorDate: Wed Mar 28 11:05:23 2018 -0700

    Refreshed branch aggregate (#10228)
---
 example/gluon/image_classification.py | 23 ++++++++++++++++++++++-
 src/engine/threaded_engine.cc         |  5 ++---
 src/engine/threaded_engine.h          |  2 +-
 src/executor/graph_executor.cc        |  5 ++---
 src/profiler/profiler.h               |  8 ++++++++
 5 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py
index 9acfda5..a67a317 100644
--- a/example/gluon/image_classification.py
+++ b/example/gluon/image_classification.py
@@ -22,6 +22,7 @@ import logging
 
 import mxnet as mx
 from mxnet import gluon
+from mxnet import profiler
 from mxnet.gluon import nn
 from mxnet.gluon.model_zoo import vision as models
 from mxnet import autograd as ag
@@ -96,6 +97,7 @@ parser.add_argument('--log-interval', type=int, default=50,
 parser.add_argument('--profile', action='store_true',
                     help='Option to turn on memory profiling for front-end, '\
                          'and prints out the memory usage by python function at the end.')
+parser.add_argument('--builtin-profiler', type=int, default=0, help='Enable built-in profiler (0=off, 1=on)')
 opt = parser.parse_args()
 
 # global variables
@@ -194,6 +196,9 @@ def train(opt, ctx):
                             kvstore = kv)
     loss = gluon.loss.SoftmaxCrossEntropyLoss()
 
+
+    total_time = 0
+    num_epochs = 0
     best_acc = [0]
     for epoch in range(opt.start_epoch, opt.epochs):
         trainer = update_learning_rate(opt.lr, trainer, epoch, opt.lr_factor, lr_steps)
@@ -223,16 +228,29 @@ def train(opt, ctx):
                                epoch, i, batch_size/(time.time()-btic), name[0], acc[0], name[1], acc[1]))
             btic = time.time()
 
+        epoch_time = time.time()-tic
+
+        # First epoch will usually be much slower than the subsequent epics,
+        # so don't factor into the average
+        if num_epochs > 0:
+          total_time = total_time + epoch_time
+        num_epochs = num_epochs + 1
+
         name, acc = metric.get()
         logger.info('[Epoch %d] training: %s=%f, %s=%f'%(epoch, name[0], acc[0], name[1], acc[1]))
-        logger.info('[Epoch %d] time cost: %f'%(epoch, time.time()-tic))
+        logger.info('[Epoch %d] time cost: %f'%(epoch, epoch_time))
         name, val_acc = test(ctx, val_data)
         logger.info('[Epoch %d] validation: %s=%f, %s=%f'%(epoch, name[0], val_acc[0], name[1], val_acc[1]))
 
         # save model if meet requirements
         save_checkpoint(epoch, val_acc[0], best_acc)
+    if num_epochs > 1:
+        print('Average epoch time: {}'.format(float(total_time)/(num_epochs - 1)))
 
 def main():
+    if opt.builtin_profiler > 0:
+        profiler.set_config(profile_all=True, aggregate_stats=True)
+        profiler.set_state('run')
     if opt.mode == 'symbolic':
         data = mx.sym.var('data')
         out = net(data)
@@ -254,6 +272,9 @@ def main():
         if opt.mode == 'hybrid':
             net.hybridize()
         train(opt, context)
+    if opt.builtin_profiler > 0:
+        profiler.set_state('stop')
+        print(profiler.dumps())
 
 if __name__ == '__main__':
     if opt.profile:
diff --git a/src/engine/threaded_engine.cc b/src/engine/threaded_engine.cc
index 77b14b4..ca5602b 100644
--- a/src/engine/threaded_engine.cc
+++ b/src/engine/threaded_engine.cc
@@ -326,8 +326,7 @@ void ThreadedEngine::PushSync(SyncFn exec_fn, Context exec_ctx,
                               FnProperty prop,
                               int priority,
                               const char* opr_name) {
-  BulkStatus& bulk_status = *BulkStatusStore::Get();
-  if (!bulk_status.bulk_size || prop != FnProperty::kNormal || priority) {
+  if (!bulk_size() || prop != FnProperty::kNormal || priority) {
     this->PushAsync([exec_fn](RunContext ctx, CallbackOnComplete on_complete) {
         exec_fn(ctx);
         on_complete();
@@ -335,9 +334,9 @@ void ThreadedEngine::PushSync(SyncFn exec_fn, Context exec_ctx,
     return;
   }
 
+  const BulkStatus& bulk_status = *BulkStatusStore::Get();
   if (bulk_status.count && exec_ctx != bulk_status.ctx) BulkFlush();
   BulkAppend(exec_fn, exec_ctx, const_vars, mutable_vars);
-  return;
 }
 
 void ThreadedEngine::DeleteVariable(SyncFn delete_fn,
diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h
index 1b9453f..d72784d 100644
--- a/src/engine/threaded_engine.h
+++ b/src/engine/threaded_engine.h
@@ -398,7 +398,7 @@ class ThreadedEngine : public Engine {
   }
 
   int bulk_size() const override {
-    return BulkStatusStore::Get()->bulk_size;
+    return profiler::Profiler::Get()->AggregateRunning() ? 0 :  BulkStatusStore::Get()->bulk_size;
   }
 
   int set_bulk_size(int bulk_size) override {
diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index ee97649..fa5931e 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -1348,7 +1348,8 @@ void GraphExecutor::InitOpSegs() {
   // Generate segments based on the graph structure
   bool prefer_bulk_exec_inference = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_INFERENCE", true);
   // Whether to perform bulk exec for training
-  bool prefer_bulk_exec = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_TRAIN", 1);
+  bool prefer_bulk_exec = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_TRAIN", 1)
+                          && !profiler::Profiler::Get()->AggregateEnabled();
 
   bool is_training = num_forward_nodes_ != total_num_nodes;
 
@@ -1359,8 +1360,6 @@ void GraphExecutor::InitOpSegs() {
   if (prefer_bulk_exec_inference && !is_training) {
     this->BulkInferenceOpSegs();
   }
-
-  return;
 }
 
 void GraphExecutor::BulkTrainingOpSegs(size_t total_num_nodes) {
diff --git a/src/profiler/profiler.h b/src/profiler/profiler.h
index 768a0bc..b8d0e8e 100644
--- a/src/profiler/profiler.h
+++ b/src/profiler/profiler.h
@@ -391,6 +391,14 @@ class Profiler {
     return aggregate_stats_.get() != nullptr;
   }
 
+  /*!
+   * \brief Whether aggregate stats are currently being recorded
+   * \return true if aggregate stats are currently being recorded
+   */
+  inline bool AggregateRunning() const {
+    return GetState() == kRunning && AggregateEnabled();
+  }
+
  public:
   /*!
    * \brief Constructor

-- 
To stop receiving notification emails like this one, please contact
jxie@apache.org.