You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by jx...@apache.org on 2018/03/28 18:05:27 UTC
[incubator-mxnet] branch master updated: Refreshed branch aggregate
(#10228)
This is an automated email from the ASF dual-hosted git repository.
jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new e5f6238 Refreshed branch aggregate (#10228)
e5f6238 is described below
commit e5f6238a2b213e70ba0ca7acc5bcf5c643502561
Author: Chris Olivier <cj...@gmail.com>
AuthorDate: Wed Mar 28 11:05:23 2018 -0700
Refreshed branch aggregate (#10228)
---
example/gluon/image_classification.py | 23 ++++++++++++++++++++++-
src/engine/threaded_engine.cc | 5 ++---
src/engine/threaded_engine.h | 2 +-
src/executor/graph_executor.cc | 5 ++---
src/profiler/profiler.h | 8 ++++++++
5 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py
index 9acfda5..a67a317 100644
--- a/example/gluon/image_classification.py
+++ b/example/gluon/image_classification.py
@@ -22,6 +22,7 @@ import logging
import mxnet as mx
from mxnet import gluon
+from mxnet import profiler
from mxnet.gluon import nn
from mxnet.gluon.model_zoo import vision as models
from mxnet import autograd as ag
@@ -96,6 +97,7 @@ parser.add_argument('--log-interval', type=int, default=50,
parser.add_argument('--profile', action='store_true',
help='Option to turn on memory profiling for front-end, '\
'and prints out the memory usage by python function at the end.')
+parser.add_argument('--builtin-profiler', type=int, default=0, help='Enable built-in profiler (0=off, 1=on)')
opt = parser.parse_args()
# global variables
@@ -194,6 +196,9 @@ def train(opt, ctx):
kvstore = kv)
loss = gluon.loss.SoftmaxCrossEntropyLoss()
+
+ total_time = 0
+ num_epochs = 0
best_acc = [0]
for epoch in range(opt.start_epoch, opt.epochs):
trainer = update_learning_rate(opt.lr, trainer, epoch, opt.lr_factor, lr_steps)
@@ -223,16 +228,29 @@ def train(opt, ctx):
epoch, i, batch_size/(time.time()-btic), name[0], acc[0], name[1], acc[1]))
btic = time.time()
+ epoch_time = time.time()-tic
+
+ # First epoch will usually be much slower than the subsequent epics,
+ # so don't factor into the average
+ if num_epochs > 0:
+ total_time = total_time + epoch_time
+ num_epochs = num_epochs + 1
+
name, acc = metric.get()
logger.info('[Epoch %d] training: %s=%f, %s=%f'%(epoch, name[0], acc[0], name[1], acc[1]))
- logger.info('[Epoch %d] time cost: %f'%(epoch, time.time()-tic))
+ logger.info('[Epoch %d] time cost: %f'%(epoch, epoch_time))
name, val_acc = test(ctx, val_data)
logger.info('[Epoch %d] validation: %s=%f, %s=%f'%(epoch, name[0], val_acc[0], name[1], val_acc[1]))
# save model if meet requirements
save_checkpoint(epoch, val_acc[0], best_acc)
+ if num_epochs > 1:
+ print('Average epoch time: {}'.format(float(total_time)/(num_epochs - 1)))
def main():
+ if opt.builtin_profiler > 0:
+ profiler.set_config(profile_all=True, aggregate_stats=True)
+ profiler.set_state('run')
if opt.mode == 'symbolic':
data = mx.sym.var('data')
out = net(data)
@@ -254,6 +272,9 @@ def main():
if opt.mode == 'hybrid':
net.hybridize()
train(opt, context)
+ if opt.builtin_profiler > 0:
+ profiler.set_state('stop')
+ print(profiler.dumps())
if __name__ == '__main__':
if opt.profile:
diff --git a/src/engine/threaded_engine.cc b/src/engine/threaded_engine.cc
index 77b14b4..ca5602b 100644
--- a/src/engine/threaded_engine.cc
+++ b/src/engine/threaded_engine.cc
@@ -326,8 +326,7 @@ void ThreadedEngine::PushSync(SyncFn exec_fn, Context exec_ctx,
FnProperty prop,
int priority,
const char* opr_name) {
- BulkStatus& bulk_status = *BulkStatusStore::Get();
- if (!bulk_status.bulk_size || prop != FnProperty::kNormal || priority) {
+ if (!bulk_size() || prop != FnProperty::kNormal || priority) {
this->PushAsync([exec_fn](RunContext ctx, CallbackOnComplete on_complete) {
exec_fn(ctx);
on_complete();
@@ -335,9 +334,9 @@ void ThreadedEngine::PushSync(SyncFn exec_fn, Context exec_ctx,
return;
}
+ const BulkStatus& bulk_status = *BulkStatusStore::Get();
if (bulk_status.count && exec_ctx != bulk_status.ctx) BulkFlush();
BulkAppend(exec_fn, exec_ctx, const_vars, mutable_vars);
- return;
}
void ThreadedEngine::DeleteVariable(SyncFn delete_fn,
diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h
index 1b9453f..d72784d 100644
--- a/src/engine/threaded_engine.h
+++ b/src/engine/threaded_engine.h
@@ -398,7 +398,7 @@ class ThreadedEngine : public Engine {
}
int bulk_size() const override {
- return BulkStatusStore::Get()->bulk_size;
+ return profiler::Profiler::Get()->AggregateRunning() ? 0 : BulkStatusStore::Get()->bulk_size;
}
int set_bulk_size(int bulk_size) override {
diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index ee97649..fa5931e 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -1348,7 +1348,8 @@ void GraphExecutor::InitOpSegs() {
// Generate segments based on the graph structure
bool prefer_bulk_exec_inference = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_INFERENCE", true);
// Whether to perform bulk exec for training
- bool prefer_bulk_exec = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_TRAIN", 1);
+ bool prefer_bulk_exec = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_TRAIN", 1)
+ && !profiler::Profiler::Get()->AggregateEnabled();
bool is_training = num_forward_nodes_ != total_num_nodes;
@@ -1359,8 +1360,6 @@ void GraphExecutor::InitOpSegs() {
if (prefer_bulk_exec_inference && !is_training) {
this->BulkInferenceOpSegs();
}
-
- return;
}
void GraphExecutor::BulkTrainingOpSegs(size_t total_num_nodes) {
diff --git a/src/profiler/profiler.h b/src/profiler/profiler.h
index 768a0bc..b8d0e8e 100644
--- a/src/profiler/profiler.h
+++ b/src/profiler/profiler.h
@@ -391,6 +391,14 @@ class Profiler {
return aggregate_stats_.get() != nullptr;
}
+ /*!
+ * \brief Whether aggregate stats are currently being recorded
+ * \return true if aggregate stats are currently being recorded
+ */
+ inline bool AggregateRunning() const {
+ return GetState() == kRunning && AggregateEnabled();
+ }
+
public:
/*!
* \brief Constructor
--
To stop receiving notification emails like this one, please contact
jxie@apache.org.