You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/11/27 17:43:15 UTC
[GitHub] kohr-h edited a comment on issue #13135: [Python] CUDNN error from
3D deconvolution
kohr-h edited a comment on issue #13135: [Python] CUDNN error from 3D deconvolution
URL: https://github.com/apache/incubator-mxnet/issues/13135#issuecomment-442150407
Update: I just ran the minimal example on the same machine under Linux (although with CUDA 10), and the error is the same:
```
>>> import mxnet as mx
>>> from mxnet import nd
>>> from mxnet.gluon import nn
>>>
>>> net = nn.Sequential()
>>> net.add(nn.Conv3D(1, (3, 3, 3), padding=(1, 1, 1)))
>>> # Without the following line, everything is okay
>>> net.add(nn.Conv3DTranspose(1, (3, 3, 3), padding=(1, 1, 1)))
>>>
>>> net.initialize(ctx=mx.gpu(0))
>>>
>>> input = nd.zeros((1, 1, 64, 64, 64), ctx=mx.gpu(0))
>>> output = net(input)
>>>
>>> output_npy = output.asnumpy()
[18:33:58] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:97: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)
[18:33:58] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:97: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)
---------------------------------------------------------------------------
MXNetError Traceback (most recent call last)
<ipython-input-1-d0c61cb28340> in <module>
13 output = net(input)
14
---> 15 output_npy = output.asnumpy()
~/git/mxnet_cu100/python/mxnet/ndarray/ndarray.py in asnumpy(self)
1978 self.handle,
1979 data.ctypes.data_as(ctypes.c_void_p),
-> 1980 ctypes.c_size_t(data.size)))
1981 return data
1982
~/git/mxnet_cu100/python/mxnet/base.py in check_call(ret)
250 """
251 if ret != 0:
--> 252 raise MXNetError(py_str(_LIB.MXGetLastError()))
253
254
MXNetError: [18:33:58] src/operator/nn/./cudnn/cudnn_deconvolution-inl.h:849: Failed to find any forward deconvolution algorithm with workspace size of 536870912 bytes, please consider reducing batch/model size or increasing the workspace size
Stack trace returned 10 entries:
[bt] (0) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(dmlc::StackTrace[abi:cxx11]()+0x53) [0x7f53485996a3]
[bt] (1) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x39) [0x7f5348599f39]
[bt] (2) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNDeconvolutionOp<float>::CuDNNAlgoSetter(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)+0x7af) [0x7f534d6d15df]
[bt] (3) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(std::_Function_handler<void (mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*,mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*), mxnet::op::CuDNNDeconvolutionOp<float>::SelectAlgo(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t)::{lambda(mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)#1}>::_M_invoke(std::_Any_data const&, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*&&, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*&&, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*&&)+0xae2) [0x7f534d6eeba2]
[bt] (4) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNAlgoReg<mxnet::op::DeconvolutionParam>::FindOrElseRegister(mxnet::op::DeconvolutionParam const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t, cudnnDataType_t, int, bool, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*, std::function<void (mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)> const&)+0x18ae) [0x7f534d6f219e]
[bt] (5) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNDeconvolutionOp<float>::SelectAlgo(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t)+0x2e1) [0x7f534d6f2aa1]
[bt] (6) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(+0x5bf6785) [0x7f534d6a2785]
[bt] (7) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(void mxnet::op::DeconvolutionCompute<mshadow::gpu>(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&)+0x625) [0x7f534d6a69b5]
[bt] (8) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::imperative::PushFCompute(std::function<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&)> const&, nnvm::Op const*, nnvm::NodeAttrs const&, mxnet::Context const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::Resource, std::allocator<mxnet::Resource> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&,std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&)::{lambda(mxnet::RunContext)#1}::operator()(mxnet::RunContext) const+0x244) [0x7f534b0cb5f4]
[bt] (9) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(+0x3be8c1b) [0x7f534b694c1b]
```
~~I'll remove the Windows label.~~ I guess I can't.
@mxnet-label-bot remove [Windows]
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services