You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2017/10/25 05:49:18 UTC
[GitHub] feevos opened a new issue #8419: DeferredInitializationError after calling trainer.step
feevos opened a new issue #8419: DeferredInitializationError after calling trainer.step
URL: https://github.com/apache/incubator-mxnet/issues/8419
## Description
Dear all, am (completely) new to mxnet. I am trying to implement a simple [unet](https://arxiv.org/abs/1505.04597) architecture for semantic segmentation with mxnet.
[](url)
This is my unet implementation:
```Python
class UNet(Block):
def __init__(self,**kwargs):
super(UNet, self).__init__(**kwargs)
self.Nfilters = 32
self.Nbatch = 1 # to be addressed later
self.NClasses = 11
self.img_size = 192
self.axis = 1 # For NCHW format
with self.name_scope():
# 32
self.conv11=gluon.nn.Conv2D(self.Nfilters,kernel_size=3,padding=1,activation='relu')
self.batch11=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv12=gluon.nn.Conv2D(self.Nfilters,kernel_size=3,padding=1,activation='relu')
self.batch12=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.pool1 = gluon.nn.MaxPool2D(pool_size=2,strides=2)
# 64
self.conv21=gluon.nn.Conv2D(self.Nfilters*2,kernel_size=3,padding=1,activation='relu')
self.batch21=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv22=gluon.nn.Conv2D(self.Nfilters*2,kernel_size=3,padding=1,activation='relu')
self.batch22=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.pool2 = gluon.nn.MaxPool2D(pool_size=2,strides=2)
# 128
self.conv31=gluon.nn.Conv2D(self.Nfilters*2**2,kernel_size=3,padding=1,activation='relu')
self.batch31=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv32=gluon.nn.Conv2D(self.Nfilters*2**2,kernel_size=3,padding=1,activation='relu')
self.batch32=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.pool3 = gluon.nn.MaxPool2D(pool_size=2,strides=2)
# 256
self.conv41=gluon.nn.Conv2D(self.Nfilters*2**3,kernel_size=3,padding=1,activation='relu')
self.batch41=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv42=gluon.nn.Conv2D(self.Nfilters*2**3,kernel_size=3,padding=1,activation='relu')
self.batch42=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.pool4 = gluon.nn.MaxPool2D(pool_size=self.axis,strides=2)
# 512
self.conv51=gluon.nn.Conv2D(self.Nfilters*2**4,kernel_size=3,padding=1,activation='relu')
self.batch51=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv52=gluon.nn.Conv2D(self.Nfilters*2**4,kernel_size=3,padding=1,activation='relu')
self.batch52=gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
#self.pool5 = gluon.nn.MaxPool2D(pool_size=2,strides=1,layout='NHWC')
self.up6 = gluon.nn.Conv2DTranspose(self.Nfilters*2**3,kernel_size=(2,2), strides=(2,2)) # Will go in to the
self.conv61 = gluon.nn.Conv2D(self.Nfilters*2**3,kernel_size=3,padding=1,activation='relu')
self.batch61= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv62 = gluon.nn.Conv2D(self.Nfilters*2**3,kernel_size=3,padding=1,activation='relu')
self.batch62= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.up7 = gluon.nn.Conv2DTranspose(self.Nfilters*2**2,kernel_size=(2,2), strides=(2,2))
self.conv71 = gluon.nn.Conv2D(self.Nfilters*2**2,kernel_size=3,padding=1,activation='relu')
self.batch71= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv72 = gluon.nn.Conv2D(self.Nfilters*2**2,kernel_size=3,padding=1,activation='relu')
self.batch72= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.up8 = gluon.nn.Conv2DTranspose(self.Nfilters*2**1,kernel_size=(2,2), strides=(2,2))
self.conv81 = gluon.nn.Conv2D(self.Nfilters*2**1,kernel_size=3,padding=1,activation='relu')
self.batch81= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv82 = gluon.nn.Conv2D(self.Nfilters*2**1,kernel_size=3,padding=1,activation='relu')
self.batch82= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.up9 = gluon.nn.Conv2DTranspose(self.Nfilters*2**1,kernel_size=(2,2), strides=(2,2))
self.conv91 = gluon.nn.Conv2D(self.Nfilters*2**1,kernel_size=3,padding=1,activation='relu')
self.batch91= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv92 = gluon.nn.Conv2D(self.Nfilters*2**1,kernel_size=3,padding=1,activation='relu')
self.batch92= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.up10 = gluon.nn.Conv2DTranspose(self.Nfilters*2**0,kernel_size=(2,2), strides=(2,2))
self.conv101 = gluon.nn.Conv2D(self.Nfilters*2**0,kernel_size=3,padding=1,activation='relu')
self.batch101= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.conv102 = gluon.nn.Conv2D(self.Nfilters*2**0,kernel_size=3,padding=1,activation='relu')
self.batch102= gluon.nn.BatchNorm(axis=self.axis, center=True, scale=True)
self.convLast = gluon.nn.Conv2D(self.NClasses,kernel_size=(1,1),padding=0)
def forward(self,x):
conv1 = self.conv11(x)
conv1 = self.batch11(conv1)
conv1 = self.conv12(conv1)
conv1 = self.batch12(conv1)
pool1 = self.pool1(conv1)
conv2 = self.conv21(pool1)
conv2 = self.batch21(conv2)
conv2 = self.conv22(conv2)
conv2 = self.batch22(conv2)
pool2 = self.pool2(conv2)
conv3 = self.conv31(pool2)
conv3 = self.batch31(conv3)
conv3 = self.conv32(conv3)
conv3 = self.batch32(conv3)
pool3 = self.pool3(conv3)
conv4 = self.conv41(pool3)
conv4 = self.batch41(conv4)
conv4 = self.conv42(conv4)
conv4 = self.batch42(conv4)
pool4 = self.pool4(conv4)
conv5 = self.conv51(pool4)
conv5 = self.batch51(conv5)
conv5 = self.conv52(conv5)
conv5 = self.batch52(conv5)
# UpSampling with transposed Convolution
conv6 = self.up6(conv5)
conv6 = mx.nd.concat(conv6,conv4)
conv6 = self.conv61(conv6)
conv6 = self.batch61(conv6)
conv6 = self.conv62(conv6)
conv6 = self.batch62(conv6)
# UpSampling with transposed Convolution
conv7 = self.up7(conv6)
conv7 = mx.nd.concat(conv7,conv3)
conv7 = self.conv71(conv7)
conv7 = self.batch71(conv7)
conv7 = self.conv72(conv7)
conv7 = self.batch72(conv7)
# UpSampling with transposed Convolution
conv8 = self.up8(conv7)
conv8 = mx.nd.concat(conv8,conv2)
conv8 = self.conv81(conv8)
conv8 = self.batch81(conv8)
conv8 = self.conv82(conv8)
conv8 = self.batch82(conv8)
# UpSampling with transposed Convolution
conv9 = self.up9(conv8)
conv9 = mx.nd.concat(conv9,conv1)
conv9 = self.conv91(conv9)
conv9 = self.batch91(conv9)
conv9 = self.conv92(conv9)
conv9 = self.batch92(conv9)
final_layer = self.convLast(conv9)
# Experimental
final_layer = nd.reshape(final_layer,(self.Nbatch,self.NClasses*self.img_size*self.img_size))
final_layer = nd.SoftmaxActivation(final_layer)
final_layer = nd.reshape(final_layer,(self.Nbatch,self.NClasses,self.img_size,self.img_size))
return final_layer
```
Upon trying to call trainer.step(Nbatch) in the following code (including only relevant blocks):
```Python
ctx = mx.gpu()
unet = UNet()
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss(axis=1,sparse_label=False)
unet.collect_params().initialize(mx.init.Xavier(magnitude=2.24),ctx=ctx)
trainer = gluon.Trainer(unet.collect_params(),'sgd',{'learning_rate':0.001})
# .... various definitions for data generator, specific training params etc.
for epoch in range(epochs):
mygen = dstl_data_generator(read_dirs,NNbatch,shape_img, 11)
generator = mygen.generator()
for it in range(mygen.len):
data,label = next(generator) # They "live" in ctx by definition in generator
with autograd.record():
output = unet(data)
loss = softmax_cross_entropy(output,label)
loss.backward()
# #################################################
# This is the part that produces the error and I don't know why
trainer.step(Nbatch) # Batch size
# #################################################
```
I get an error which I do not understand. Any ideas on how I can fix this? Thank you very much
## Error Message:
```Python
---------------------------------------------------------------------------
DeferredInitializationError Traceback (most recent call last)
<ipython-input-23-06e4be1b8fb6> in <module>()
10 loss = softmax_cross_entropy(output,label)
11 loss.backward()
---> 12 trainer.step(NNbatch) # Batch size
13
14 curr_loss=nd.mean(loss).asscalar()
/home/dia021/anaconda2/lib/python2.7/site-packages/mxnet/gluon/trainer.pyc in step(self, batch_size, ignore_stale_grad)
128 """
129 if not self._kv_initialized:
--> 130 self._init_kvstore()
131
132 self._optimizer.rescale_grad = self._scale / batch_size
/home/dia021/anaconda2/lib/python2.7/site-packages/mxnet/gluon/trainer.pyc in _init_kvstore(self)
94
95 def _init_kvstore(self):
---> 96 arg_arrays = {param.name: param.data(self._contexts[0]) for param in self._params}
97 kvstore, update_on_kvstore = _create_kvstore(self._kvstore, len(self._contexts),
98 arg_arrays)
/home/dia021/anaconda2/lib/python2.7/site-packages/mxnet/gluon/trainer.pyc in <dictcomp>((param,))
94
95 def _init_kvstore(self):
---> 96 arg_arrays = {param.name: param.data(self._contexts[0]) for param in self._params}
97 kvstore, update_on_kvstore = _create_kvstore(self._kvstore, len(self._contexts),
98 arg_arrays)
/home/dia021/anaconda2/lib/python2.7/site-packages/mxnet/gluon/parameter.pyc in data(self, ctx)
334 else:
335 ctx = context.current_context()
--> 336 self._check_initialized(ctx)
337 return self._data[ctx]
338
/home/dia021/anaconda2/lib/python2.7/site-packages/mxnet/gluon/parameter.pyc in
_check_initialized(self, ctx)
139 return
140 if self._deferred_init:
--> 141 raise DeferredInitializationError
142 raise RuntimeError(
143 "Parameter %s has not been initialized. Note that " \
DeferredInitializationError:
----------------------------------------------------------------
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services