You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by gi...@git.apache.org on 2017/10/07 14:07:35 UTC
[GitHub] shipeng-uestc opened a new issue #8170: I can't run the mnist tutorial by GPU
shipeng-uestc opened a new issue #8170: I can't run the mnist tutorial by GPU
URL: https://github.com/apache/incubator-mxnet/issues/8170
For bugs or installation issues, please provide the following information.
The more information you provide, the more likely people will be able to help you.
## Environment info
Operating System:
Ubuntu16.04
Compiler:
Package used (Python/R/Scala/Julia):
Python3.6
MXNet version:
MXNet 0.11.0
Or if installed from source:
MXNet commit hash (`git rev-parse HEAD`):
If you are using python package, please provide
Python version and distribution:
python3.6
If you are using R package, please provide
R `sessionInfo()`:
## Error Message:
Please paste the full error message, including stack trace.
MXNetError Traceback (most recent call last)
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/symbol/symbol.py in simple_bind(self, ctx, grad_req, type_dict, stype_dict, group2ctx, shared_arg_names, shared_exec, shared_buffer, **kwargs)
1483 shared_exec_handle,
-> 1484 ctypes.byref(exe_handle)))
1485 except MXNetError as e:
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/base.py in check_call(ret)
144 if ret != 0:
--> 145 raise MXNetError(py_str(_LIB.MXGetLastError()))
146
MXNetError: [21:55:13] src/storage/storage.cc:109: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading CUDA: unknown error
Stack trace returned 10 entries:
[bt] (0) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x230aac) [0x7faddfdcbaac]
[bt] (1) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a15ab7) [0x7fade15b0ab7]
[bt] (2) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a18239) [0x7fade15b3239]
[bt] (3) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a187bc) [0x7fade15b37bc]
[bt] (4) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x153b691) [0x7fade10d6691]
[bt] (5) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1558f04) [0x7fade10f3f04]
[bt] (6) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154061a) [0x7fade10db61a]
[bt] (7) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1544737) [0x7fade10df737]
[bt] (8) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154c52a) [0x7fade10e752a]
[bt] (9) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154cc24) [0x7fade10e7c24]
During handling of the above exception, another exception occurred:
RuntimeError Traceback (most recent call last)
<ipython-input-4-02e34f82ecc4> in <module>()
8 eval_metric='acc',
9 batch_end_callback = mx.callback.Speedometer(batch_size, 100),
---> 10 num_epoch=10)
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/module/base_module.py in fit(self, train_data, eval_data, eval_metric, epoch_end_callback, batch_end_callback, kvstore, optimizer, optimizer_params, eval_end_callback, eval_batch_end_callback, initializer, arg_params, aux_params, allow_missing, force_rebind, force_init, begin_epoch, num_epoch, validation_metric, monitor)
458
459 self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label,
--> 460 for_training=True, force_rebind=force_rebind)
461 if monitor is not None:
462 self.install_monitor(monitor)
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/module/module.py in bind(self, data_shapes, label_shapes, for_training, inputs_need_grad, force_rebind, shared_module, grad_req)
415 fixed_param_names=self._fixed_param_names,
416 grad_req=grad_req,
--> 417 state_names=self._state_names)
418 self._total_exec_bytes = self._exec_group._total_exec_bytes
419 if shared_module is not None:
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/module/executor_group.py in __init__(self, symbol, contexts, workload, data_shapes, label_shapes, param_names, for_training, inputs_need_grad, shared_group, logger, fixed_param_names, grad_req, state_names)
229 self.num_outputs = len(self.symbol.list_outputs())
230
--> 231 self.bind_exec(data_shapes, label_shapes, shared_group)
232
233 def decide_slices(self, data_shapes):
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/module/executor_group.py in bind_exec(self, data_shapes, label_shapes, shared_group, reshape)
325 else:
326 self.execs.append(self._bind_ith_exec(i, data_shapes_i, label_shapes_i,
--> 327 shared_group))
328
329 self.data_shapes = data_shapes
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/module/executor_group.py in _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group)
601 type_dict=input_types, shared_arg_names=self.param_names,
602 shared_exec=shared_exec,
--> 603 shared_buffer=shared_data_arrays, **input_shapes)
604 self._total_exec_bytes += int(executor.debug_str().split('\n')[-3].split()[1])
605 return executor
~/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/symbol/symbol.py in simple_bind(self, ctx, grad_req, type_dict, stype_dict, group2ctx, shared_arg_names, shared_exec, shared_buffer, **kwargs)
1488 error_msg += "%s: %s\n" % (k, v)
1489 error_msg += "%s" % e
-> 1490 raise RuntimeError(error_msg)
1491
1492 # update shared_buffer
RuntimeError: simple_bind error. Arguments:
data: (100, 1, 28, 28)
softmax_label: (100,)
[21:55:13] src/storage/storage.cc:109: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading CUDA: unknown error
Stack trace returned 10 entries:
[bt] (0) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x230aac) [0x7faddfdcbaac]
[bt] (1) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a15ab7) [0x7fade15b0ab7]
[bt] (2) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a18239) [0x7fade15b3239]
[bt] (3) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1a187bc) [0x7fade15b37bc]
[bt] (4) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x153b691) [0x7fade10d6691]
[bt] (5) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1558f04) [0x7fade10f3f04]
[bt] (6) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154061a) [0x7fade10db61a]
[bt] (7) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x1544737) [0x7fade10df737]
[bt] (8) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154c52a) [0x7fade10e752a]
[bt] (9) /home/shipeng/anaconda3/envs/gluon/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x154cc24) [0x7fade10e7c24]
## Minimum reproducible example
if you are using your own code, please provide a short script that reproduces the error.
`import mxnet as mx
mnist = mx.test_utils.get_mnist()
batch_size = 100
train_iter = mx.io.NDArrayIter(mnist['train_data'], mnist['train_label'], batch_size, shuffle=True)
val_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size)
data = mx.sym.var('data')
# first conv layer
conv1 = mx.sym.Convolution(data=data, kernel=(5,5), num_filter=20)
tanh1 = mx.sym.Activation(data=conv1, act_type="tanh")
pool1 = mx.sym.Pooling(data=tanh1, pool_type="max", kernel=(2,2), stride=(2,2))
# second conv layer
conv2 = mx.sym.Convolution(data=pool1, kernel=(5,5), num_filter=50)
tanh2 = mx.sym.Activation(data=conv2, act_type="tanh")
pool2 = mx.sym.Pooling(data=tanh2, pool_type="max", kernel=(2,2), stride=(2,2))
# first fullc layer
flatten = mx.sym.flatten(data=pool2)
fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=500)
tanh3 = mx.sym.Activation(data=fc1, act_type="tanh")
# second fullc
fc2 = mx.sym.FullyConnected(data=tanh3, num_hidden=10)
# softmax loss
lenet = mx.sym.SoftmaxOutput(data=fc2, name='softmax')
# create a trainable module on GPU 0
lenet_model = mx.mod.Module(symbol=lenet, context=mx.gpu())
# train with the same
lenet_model.fit(train_iter,
eval_data=val_iter,
optimizer='sgd',
optimizer_params={'learning_rate':0.1},
eval_metric='acc',
batch_end_callback = mx.callback.Speedometer(batch_size, 100),
num_epoch=10)
`
## Steps to reproduce
or if you are running standard examples, please provide the commands you have run that lead to the error.
1.
2.
3.
## What have you tried to solve it?
1.
2.
3.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services