[GitHub] thomasmooon opened a new issue #7490: MXNet R: How apply dropout to input using mx.nd.Droput() and mx.mlp()?

[gi...@git.apache.org]

thomasmooon opened a new issue #7490: MXNet R: How apply dropout to input using mx.nd.Droput() and mx.mlp()?
URL: https://github.com/apache/incubator-mxnet/issues/7490
 
 
   I fail in using `mx.nd.Dropout()` given the minimum reproducible Example_1 attached below.
   
   - As one can see the output of _data_ and _data.withDropout_ is the same. 
   - In addition, Using `mx.mlp(data=data, label=data.withDropout,...) `compared to` mx.mlp(data=data, label=data,...) ` results in the same Train-rmse.
   
   **What is my mistake?**
   
   As a workaround I manually perform custom dropout to my data as like as in Example_2 with function `Dropout()`.
   
   # Example_1: mx.nd.Dropout()
   ```
   library(mxnet)
   
   # create example data
   set.seed(1)
   data <- matrix(floor(runif(10,1,10)),ncol=2)
   data <- mx.nd.array(data)
   
   # apply mx.nd.Dropout()
   data.withDroput <- mx.nd.Dropout(data = data, p = 0.5)
   
   # compare data VS data.withDroput
   data
   #       [,1] [,2]
   # [1,]    3    9
   # [2,]    4    9
   # [3,]    6    6
   # [4,]    9    6
   # [5,]    2    1
   data.withDroput
   #       [,1] [,2]
   # [1,]    3    9
   # [2,]    4    9
   # [3,]    6    6
   # [4,]    9    6
   # [5,]    2    1
   
   # train: data = data, label = data.withDroput ####
   mx.set.seed(1)
   
   model <- mx.mlp(
     data = data,
     label = data.withDroput,
     hidden_node = c(100, 100, 100),
     array.layout = "rowmajor",
     activation = "tanh",
     out_node = nrow(data),
     out_activation = "rmse",
     num.round = 10,
     learning.rate = 0.1,
     momentum = 0.1,
     eval.metric = mx.metric.rmse,
     initializer = mx.init.uniform(0.1),
     epoch.end.callback = mx.callback.log.train.metric(100)
   )
   # Start training with 1 devices
   # [1] Train-rmse=NaN
   # [2] Train-rmse=2.77657693899481
   # [3] Train-rmse=2.65975473627393
   # [4] Train-rmse=2.52975210749077
   # [5] Train-rmse=2.36495027661254
   # [6] Train-rmse=2.15330790153018
   # [7] Train-rmse=1.90352516425587
   # [8] Train-rmse=1.64214887494812
   # [9] Train-rmse=1.40092835645985
   # [10] Train-rmse=1.20226865114326
   
   # train: data = data, label = data ####
   mx.set.seed(1)
   
   model <- mx.mlp(
     data = data,
     label = data,
     hidden_node = c(100, 100, 100),
     array.layout = "rowmajor",
     activation = "tanh",
     out_node = nrow(data),
     out_activation = "rmse",
     num.round = 10,
     learning.rate = 0.1,
     momentum = 0.1,
     eval.metric = mx.metric.rmse,
     initializer = mx.init.uniform(0.1),
     epoch.end.callback = mx.callback.log.train.metric(100)
   )
   # Start training with 1 devices
   # [1] Train-rmse=NaN
   # [2] Train-rmse=2.77657693899481
   # [3] Train-rmse=2.65975473627393
   # [4] Train-rmse=2.52975210749077
   # [5] Train-rmse=2.36495027661254
   # [6] Train-rmse=2.15330790153018
   # [7] Train-rmse=1.90352516425587
   # [8] Train-rmse=1.64214887494812
   # [9] Train-rmse=1.40092835645985
   # [10] Train-rmse=1.20226865114326
   
   ```
   # Example_2: custom dropout function
   ```
   library(mxnet)
   
   # create example data ####
   set.seed(1)
   data <- matrix(floor(runif(10,1,10)),ncol=2)
   data <- mx.nd.array(data)
   
   # some kind of custom dropout ####
   Dropout <- function(x,p) {
     n <- floor(p*length(x))
     idx <- sample(1:length(x),n)
     x[idx] <- 0
     return(x)
   }
   set.seed(2)
   data.withDroput <- apply(data,2, function(x) Dropout(x, 0.5))
   
   # compare data VS data.withDroput ####
   
   data
   
   #       [,1] [,2]
   # [1,]    3    9
   # [2,]    4    9
   # [3,]    6    6
   # [4,]    9    6
   # [5,]    2    1
   
   data.withDroput
   
   #       [,1] [,2]
   # [1,]    0    0
   # [2,]    4    9
   # [3,]    0    0
   # [4,]    9    6
   # [5,]    2    1
   
   # train: data = data, label = data.withDroput ####
   mx.set.seed(1)
   
   model <- mx.mlp(
     data = data,
     label = data.withDroput,
     hidden_node = c(100, 100, 100),
     array.layout = "rowmajor",
     activation = "tanh",
     out_node = nrow(data),
     out_activation = "rmse",
     num.round = 10,
     learning.rate = 0.1,
     momentum = 0.1,
     eval.metric = mx.metric.rmse,
     initializer = mx.init.uniform(0.1),
     epoch.end.callback = mx.callback.log.train.metric(100)
   )
   
   
   # Start training with 1 devices
   # [1] Train-rmse=NaN
   # [2] Train-rmse=2.11040204294938
   # [3] Train-rmse=2.02803825337813
   # [4] Train-rmse=1.94056382233993
   # [5] Train-rmse=1.84112715113636
   # [6] Train-rmse=1.72132630806252
   # [7] Train-rmse=1.57952232263995
   # [8] Train-rmse=1.42125792291129
   # [9] Train-rmse=1.25770766817279
   # [10] Train-rmse=1.10197211192421
   
   ```
   It's not surprising, that the Train-rmse differs compared to Example_1.
 
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services