[GitHub] chowkamlee81 commented on issue #9143: maximum value of activation function using tanh goes beyoynd maximum value...

[GitBox <gi...@apache.org>]

chowkamlee81 commented on issue #9143: maximum value of activation function using tanh goes beyoynd maximum value...
URL: https://github.com/apache/incubator-mxnet/issues/9143#issuecomment-352949248
 
 
   Thanks for your kind reply. iam posting the snippet code below for GAN generator development for future frame prediction taking past few 4 frames as input.
   
   # load training data
       train_data = TrainDataLoader(segdb, config, batch_size=input_batch_size, crop_height=config.TRAIN.CROP_HEIGHT, crop_width=config.TRAIN.CROP_WIDTH,
                                    shuffle=config.TRAIN.SHUFFLE, ctx=ctx)
   
       # infer max shape
       max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))),
                         ('data_prevOne', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in config.SCALES]),max([v[1] for v in config.SCALES]))),
                         ('data_prevTwo', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in config.SCALES]),max([v[1] for v in config.SCALES]))),
                         ('data_prevThree', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in config.SCALES]),max([v[1] for v in config.SCALES])))]
       max_label_shape = [('label', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]
   
       data_shape_dict = dict(max_data_shape)
   
       data = mx.symbol.Variable(name="data")
       data_prevOne = mx.symbol.Variable(name="data_prevOne")
       data_prevTwo = mx.symbol.Variable(name="data_prevTwo")
       data_prevThree = mx.symbol.Variable(name="data_prevThree")
       seg_cls_gt = mx.symbol.Variable(name='label')
       data_by_2 = mx.symbol.Convolution(name='data_by_2', data=data, num_filter=1, pad=(0, 0), kernel=(1, 1),stride=(2, 2), no_bias=True)
       data_prevOne_by_2 = mx.symbol.Convolution(name='data_prevOne_by_2', data=data_prevOne, num_filter=1, pad=(0, 0),kernel=(1, 1), stride=(2, 2), no_bias=True)
       data_prevTwo_by_2 = mx.symbol.Convolution(name='data_prevTwo_by_2', data=data_prevTwo, num_filter=1, pad=(0, 0),kernel=(1, 1), stride=(2, 2), no_bias=True)
       data_prevThree_by_2 = mx.symbol.Convolution(name='data_prevThree_by_2', data=data_prevThree, num_filter=1,pad=(0, 0), kernel=(1, 1), stride=(2, 2), no_bias=True)
   
       data_by_4 = mx.symbol.Convolution(name='data_by_4', data=data_by_2, num_filter=1, pad=(0, 0), kernel=(1, 1),stride=(2, 2), no_bias=True)
       data_prevOne_by_4 = mx.symbol.Convolution(name='data_prevOne_by_4', data=data_prevOne_by_2, num_filter=1,pad=(0, 0), kernel=(1, 1), stride=(2, 2), no_bias=True)
       data_prevTwo_by_4 = mx.symbol.Convolution(name='data_prevTwo_by_4', data=data_prevTwo_by_2, num_filter=1,pad=(0, 0), kernel=(1, 1), stride=(2, 2), no_bias=True)
       data_prevThree_by_4 = mx.symbol.Convolution(name='data_prevThree_by_4', data=data_prevThree_by_2, num_filter=1,pad=(0, 0), kernel=(1, 1), stride=(2, 2), no_bias=True)
   
       data_input_scale0 = mx.symbol.Concat(data_by_4, data_prevOne_by_4, data_prevTwo_by_4,data_prevThree_by_4, dim=1)
       conv1_scale0 = mx.symbol.Convolution(name='conv1_scale0', data=data_input_scale0, num_filter=128, pad=(1, 1),kernel=(3, 3), stride=(1, 1), no_bias=True)
       conv1_scale0_relu = mx.symbol.Activation(name='conv1_scale0_relu', data=conv1_scale0, act_type='tanh')
       conv2_scale0 = mx.symbol.Convolution(name='conv2_scale0', data=conv1_scale0_relu, num_filter=256, pad=(1, 1),kernel=(3, 3), stride=(1, 1), no_bias=True)
       conv2_scale0_relu = mx.symbol.Activation(name='conv2_scale0_relu', data=conv2_scale0, act_type='tanh')
       conv3_scale0 = mx.symbol.Convolution(name='conv3_scale0', data=conv2_scale0_relu, num_filter=128, pad=(1, 1),kernel=(3, 3), stride=(1, 1), no_bias=True)
       conv3_scale0_relu = mx.symbol.Activation(name='conv3_scale0_relu', data=conv3_scale0, act_type='tanh')
       conv4_scale0 = mx.symbol.Convolution(name='conv4_scale0', data=conv3_scale0_relu, num_filter=1, pad=(1, 1),kernel=(3, 3), stride=(1, 1), no_bias=True)
       conv4_scale0_relu = mx.symbol.Activation(name='conv4_scale0_relu', data=conv4_scale0, act_type='tanh')
       conv4_scale0_relu = mx.symbol.UpSampling(conv4_scale0_relu, num_filter=1, scale=2, sample_type='bilinear',num_args=2, name="upsampling0")
   
       heat_map = mx.symbol.UpSampling(conv4_scale0_relu, num_filter=1, scale=2, sample_type='bilinear', num_args=2,name="upsampling1")
       
       arg_names = heat_map.list_arguments()  # 'data'
       arg_shapes, _, aux_shapes = heat_map.infer_shape(**data_shape_dict)
       arg_arrays = [mx.nd.zeros(shape, ctx=mx.gpu(0)) for shape in arg_shapes]
       grad_arrays = [mx.nd.zeros(shape, ctx=mx.gpu(0)) for shape in arg_shapes]
       reqs = ["write" for name in arg_names]
       arg_map = dict(zip(arg_names, arg_arrays))
       grad_map = dict(zip(arg_names, grad_arrays))
   
   
       for name in arg_names:
           if "weight" in name:
               arr = arg_map[name]
               arr[:] = mx.rnd.uniform(-0.7, 0.7, arr.shape)
   
   
       model = heat_map.bind(ctx=mx.gpu(0), args=arg_arrays, args_grad=grad_arrays, grad_req=reqs)
       out_grad = mx.nd.zeros(model.outputs[0].shape, ctx=mx.gpu(0))
   
       num_round = 10
       for i in range(num_round):
           train_loss = 0.
           train_data.reset()
           for nbatch, data_batch in enumerate(train_data):
               arg_map["data"][:] = data_batch.data[0][0]
               arg_map["data_prevOne"][:] = data_batch.data[0][1]
               arg_map["data_prevTwo"][:] = data_batch.data[0][2]
               arg_map["data_prevThree"][:] = data_batch.data[0][3]
   
   
               data0 = arg_map["data"][:].asnumpy()
               data1 = arg_map["data_prevOne"][:].asnumpy()
               data2 = arg_map["data_prevTwo"][:].asnumpy()
               data3 = arg_map["data_prevThree"][:].asnumpy()
               val0 = ndimage.mean((data0 - data1) ** 2)
               val1 = ndimage.mean((data1 - data2) ** 2)
               val2 = ndimage.mean((data2 - data3) ** 2)
   
   
               model.forward(is_train=True)
               theta = model.outputs[0].asnumpy()
               
               pred_temp = theta.reshape(-1)
       	    max_val = np.max(pred_temp, axis=0, keepdims=True)
   
   The value of max_val i got was around 2. 031 and the value of tanh is between 1 and -1.
   Kindly suggest how to proceed further

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services