You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/11/27 17:43:15 UTC

[GitHub] kohr-h edited a comment on issue #13135: [Python] CUDNN error from 3D deconvolution

kohr-h edited a comment on issue #13135: [Python] CUDNN error from 3D deconvolution
URL: https://github.com/apache/incubator-mxnet/issues/13135#issuecomment-442150407
 
 
   Update: I just ran the minimal example on the same machine under Linux (although with CUDA 10), and the error is the same:
   ```
   >>> import mxnet as mx
   >>> from mxnet import nd
   >>> from mxnet.gluon import nn
   >>> 
   >>> net = nn.Sequential()
   >>> net.add(nn.Conv3D(1, (3, 3, 3), padding=(1, 1, 1)))
   >>> # Without the following line, everything is okay
   >>> net.add(nn.Conv3DTranspose(1, (3, 3, 3), padding=(1, 1, 1)))
   >>> 
   >>> net.initialize(ctx=mx.gpu(0))
   >>> 
   >>> input = nd.zeros((1, 1, 64, 64, 64), ctx=mx.gpu(0))
   >>> output = net(input)
   >>> 
   >>> output_npy = output.asnumpy()
   [18:33:58] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:97: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)
   [18:33:58] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:97: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)
   ---------------------------------------------------------------------------
   MXNetError                                Traceback (most recent call last)
   <ipython-input-1-d0c61cb28340> in <module>
        13 output = net(input)
        14
   ---> 15 output_npy = output.asnumpy()
   
   ~/git/mxnet_cu100/python/mxnet/ndarray/ndarray.py in asnumpy(self)
      1978             self.handle,
      1979             data.ctypes.data_as(ctypes.c_void_p),
   -> 1980             ctypes.c_size_t(data.size)))
      1981         return data
      1982
   
   ~/git/mxnet_cu100/python/mxnet/base.py in check_call(ret)
       250     """
       251     if ret != 0:
   --> 252         raise MXNetError(py_str(_LIB.MXGetLastError()))
       253
       254
   
   MXNetError: [18:33:58] src/operator/nn/./cudnn/cudnn_deconvolution-inl.h:849: Failed to find any forward deconvolution algorithm with workspace size of 536870912 bytes, please consider reducing batch/model size or increasing the workspace size
   
   Stack trace returned 10 entries:
   [bt] (0) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(dmlc::StackTrace[abi:cxx11]()+0x53) [0x7f53485996a3]
   [bt] (1) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x39) [0x7f5348599f39]
   [bt] (2) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNDeconvolutionOp<float>::CuDNNAlgoSetter(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)+0x7af) [0x7f534d6d15df]
   [bt] (3) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(std::_Function_handler<void (mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*,mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*), mxnet::op::CuDNNDeconvolutionOp<float>::SelectAlgo(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t)::{lambda(mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)#1}>::_M_invoke(std::_Any_data const&, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*&&, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*&&, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*&&)+0xae2) [0x7f534d6eeba2]
   [bt] (4) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNAlgoReg<mxnet::op::DeconvolutionParam>::FindOrElseRegister(mxnet::op::DeconvolutionParam const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t, cudnnDataType_t, int, bool, mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*, std::function<void (mxnet::op::CuDNNAlgo<cudnnConvolutionFwdAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdDataAlgo_t>*, mxnet::op::CuDNNAlgo<cudnnConvolutionBwdFilterAlgo_t>*)> const&)+0x18ae) [0x7f534d6f219e]
   [bt] (5) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::op::CuDNNDeconvolutionOp<float>::SelectAlgo(mxnet::RunContext const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, std::vector<nnvm::TShape, std::allocator<nnvm::TShape> > const&, cudnnDataType_t, cudnnDataType_t)+0x2e1) [0x7f534d6f2aa1]
   [bt] (6) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(+0x5bf6785) [0x7f534d6a2785]
   [bt] (7) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(void mxnet::op::DeconvolutionCompute<mshadow::gpu>(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&)+0x625) [0x7f534d6a69b5]
   [bt] (8) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(mxnet::imperative::PushFCompute(std::function<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::TBlob, std::allocator<mxnet::TBlob> > const&)> const&, nnvm::Op const*, nnvm::NodeAttrs const&, mxnet::Context const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::Resource, std::allocator<mxnet::Resource> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&,std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&)::{lambda(mxnet::RunContext)#1}::operator()(mxnet::RunContext) const+0x244) [0x7f534b0cb5f4]
   [bt] (9) /home/hkohr/git/mxnet_cu100/python/mxnet/../../lib/libmxnet.so(+0x3be8c1b) [0x7f534b694c1b]
   
   ```
   
   ~~I'll remove the Windows label.~~ I guess I can't.
   @mxnet-label-bot remove [Windows]

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services