You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by pr...@apache.org on 2017/05/12 21:05:41 UTC
incubator-systemml git commit: [SYSTEMML-1483] Add Deconvolution
layer in nn library.
Repository: incubator-systemml
Updated Branches:
refs/heads/master a3e7e5c49 -> d04d2381f
[SYSTEMML-1483] Add Deconvolution layer in nn library.
Closes #490.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d04d2381
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d04d2381
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d04d2381
Branch: refs/heads/master
Commit: d04d2381f369bc29c4c33e98381bcdc8a4d0aebb
Parents: a3e7e5c
Author: prithvirajsen <se...@us.ibm.com>
Authored: Fri May 12 13:59:20 2017 -0700
Committer: prithvirajsen <se...@us.ibm.com>
Committed: Fri May 12 14:04:15 2017 -0700
----------------------------------------------------------------------
scripts/nn/layers/conv2d_transpose.dml | 248 ++++++++++++++++++++++++++++
scripts/nn/test/grad_check.dml | 109 ++++++++++++
scripts/nn/test/run_tests.dml | 2 +
scripts/nn/test/test.dml | 37 +++++
4 files changed, 396 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d04d2381/scripts/nn/layers/conv2d_transpose.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/layers/conv2d_transpose.dml b/scripts/nn/layers/conv2d_transpose.dml
new file mode 100644
index 0000000..ecd7a1c
--- /dev/null
+++ b/scripts/nn/layers/conv2d_transpose.dml
@@ -0,0 +1,248 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/*
+ * 2D Transpose convolutional layer.
+ *
+ * Utilizes built-in convolution operators for higher performance.
+ */
+
+forward = function(matrix[double] X, matrix[double] W, matrix[double] b,
+ int C, int Hin, int Win, int Hf, int Wf,
+ int strideh, int stridew, int padh, int padw,
+ int out_padh, int out_padw)
+ return (matrix[double] out, int Hout, int Wout){
+ /*
+ * Computes the forward pass for a 2D spatial transpose convolutional
+ * layer with F filters. The input data has N examples, each
+ * represented as a 3D tensor flattened into a single vector.
+ *
+ * Inputs:
+ * - X: Inputs, of shape (N, C*Hin*Win).
+ * - W: Weights, of shape (F, C*Hf*Wf).
+ * - b: Biases, of shape (F, 1).
+ * - C: Number of input channels (dimensionality of depth).
+ * - Hin: Input height.
+ * - Win: Input width.
+ * - Hf: Filter height.
+ * - Wf: Filter width.
+ * - strideh: Stride over height.
+ * - stridew: Stride over width.
+ * - padh: Padding for top and bottom sides.
+ * - padw: Padding for left and right sides.
+ * - out_padh: extra padding for top side. This should
+ * lie in [0, strideh-1].
+ * - out_padw: extra padding for right side. This should
+ * lie in [0, stridew-1].
+ *
+ * Outputs:
+ * - out: Outputs, of shape (N, F*Hout*Wout).
+ * - Hout: Output height.
+ * - Wout: Output width.
+ */
+ N = nrow(X)
+ F = nrow(W)
+ Hout = strideh * (Hin-1) - 2*padh + Hf + out_padh
+ Wout = stridew * (Win-1) - 2*padw + Wf + out_padw
+
+ /*
+ * Transpose convolution aims to go in the other direction of
+ * (direct) convolution, i.e., given input X, produce output O such
+ * that running convolution on O recovers X. This is achieved by
+ * conv2d_backward_data (since the derivative wrt data must produce
+ * output of same size as the input to conv2d). By reusing a built-in
+ * operator we achieve efficiency and restrict the number of built-in
+ * operators to manageable levels. Plus, most other deep-learning
+ * packages make use of the same strategy which means this
+ * implementation of transpose convolution is 'in-sync' with them.
+ *
+ * One potential downside of reusing conv2d_backward_data is the fact
+ * that it rotates the filter by 180 degrees before applying it. This
+ * needs to be kept in mind when interpreting the output of transpose
+ * convolution.
+ */
+ out = conv2d_backward_data(W, X, stride=[strideh,stridew], padding=[padh,padw],
+ input_shape=[N,C,Hout,Wout], filter_shape=[F,C,Hf,Wf])
+
+ out = bias_add(out, b)
+}
+
+backward = function(matrix[double] dout, int Hout, int Wout,
+ matrix[double] X, matrix[double] W, matrix[double] b,
+ int C, int Hin, int Win, int Hf, int Wf,
+ int strideh, int stridew, int padh, int padw)
+ return (matrix[double] dX, matrix[double] dW, matrix[double] db){
+ /*
+ * Computes the backward pass for a 2D spatial transpose
+ * convolutional layer with F filters.
+ *
+ * Inputs:
+ * - dout: Gradient wrt `out` from upstream, of
+ * shape (N, F*Hout*Wout).
+ * - Hout: Output height.
+ * - Wout: Output width.
+ * - X: Inputs, of shape (N, C*Hin*Win).
+ * - W: Weights, of shape (F, C*Hf*Wf).
+ * - b: Biases, of shape (F, 1).
+ * - C: Number of input channels (dimensionality of depth).
+ * - Hin: Input height.
+ * - Win: Input width.
+ * - Hf: Filter height.
+ * - Wf: Filter width.
+ * - strideh: Stride over height.
+ * - stridew: Stride over width.
+ * - padh: Padding for top and bottom sides.
+ * - padw: Padding for left and right sides.
+ *
+ * Outputs:
+ * - dX: Gradient wrt `X`, of shape (N, C*Hin*Win).
+ * - dW: Gradient wrt `W`, of shape (F, C*Hf*Wf).
+ * - db: Gradient wrt `b`, of shape (F, 1).
+ */
+ N = nrow(X)
+ F = nrow(W)
+
+ /*
+ * conv2d_backward_filter takes the input and delta map as first and
+ * second args, respectively. Given that, we need to compute the
+ * grad (wrt to filter) for transpose convolution where the roles of
+ * the input and output are reversed, we reverse the order of the
+ * args (along with setting input_shape to the delta map shape).
+ * Effectively, we are running a direct convolution with X as the
+ * filter and the dout as the input. To convince oneself that the
+ * interconnections between the cells of the filter, input and delta
+ * map are preserved please keep in mind that the forward of
+ * convolution transpose rotates the filter by 180 degrees before
+ * applying it.
+ */
+ dW = conv2d_backward_filter(dout, X, stride=[strideh,stridew], padding=[padh,padw],
+ input_shape=[N,C,Hout,Wout], filter_shape=[F,C,Hf,Wf])
+
+ /*
+ * Since the forward for transpose convolution makes a call to
+ * conv2d_backward_data, to compute its derivative wrt to data
+ * we can run conv2d by applying the filter on the delta
+ * map (this makes sense because convolution transpose is the
+ * 'reverse' of convolution). Its easy to see that this will produce
+ * output of the required size. To convince oneself that conv2d will
+ * respect the interconnections between the cells in the delta map
+ * and the filter, keep in mind that the forward function rotates the
+ * filter by 180 degrees before applying it.
+ */
+ dX = conv2d(dout, W, input_shape=[N,C,Hout,Wout], filter_shape=[F,C,Hf,Wf],
+ stride=[strideh,stridew], padding=[padh,padw])
+
+ db = rowSums(matrix(colSums(dout), rows=F, cols=Hout*Wout))
+}
+
+init = function(int F, int C, int Hf, int Wf)
+ return (matrix[double] W, matrix[double] b){
+ /*
+ * Utility function to initialize the parameters of this layer.
+ *
+ * We use the heuristic by He et al., which limits the magnification
+ * of inputs/gradients during forward/backward passes by scaling
+ * unit-Gaussian weights by a factor of sqrt(2/n), under the
+ * assumption of relu neurons.
+ * - http://arxiv.org/abs/1502.01852
+ *
+ * Inputs:
+ * - F: Number of filters.
+ * - C: Number of input channels (dimensionality of depth).
+ * - Hf: Filter height.
+ * - Wf: Filter width.
+ *
+ * Outputs:
+ * - W: Weights, of shape (F, C*Hf*Wf).
+ * - b: Biases, of shape (F, 1).
+ */
+ W = rand(rows=F, cols=C*Hf*Wf, pdf="normal") * sqrt(2/(C*Hf*Wf))
+ b = matrix(0, rows=F, cols=1)
+}
+
+init_bilinear = function(int C, int K)
+ return (matrix[double] W, matrix[double] b){
+ /*
+ * Utility function to upsample using this layer.
+ *
+ * Upsampling the input by factor f (each side) requires
+ * channel-wise independent kernels of size K = 2f - f%2,
+ * stride = f and pad = ceil((f-1)/2). The weights are set
+ * via bilinear interpolation, bias is set to 0.
+ *
+ * Inputs:
+ * - C: Number of input channels (dimensionality of depth).
+ * - K: Kernel size (upsampling requires a square filter
+ * of size K X K).
+ *
+ * Outputs:
+ * - W: Weights, of shape (C, C*K*K).
+ * - b: Biases, of shape (C, 1).
+ */
+ factor_up = ceil(K / 2)
+ center = (2 * factor_up - factor_up %% 2 - 1) / 2 / factor_up
+ vect = 1 - abs(seq(0, K-1) / factor_up - center)
+ weights = matrix(vect %*% t(vect), rows=1, cols=K*K)
+
+ /*
+ * To create a multi-channel channel-independent upsampling filter,
+ * we need to intersperse the filter weights with 0s. For instance,
+ * consider the case of 2X upsampling. In this case, K=4 and we have
+ * K^2=16 weights to include into the 3D tensor representing the
+ * filter which should look like the following (assuming 3 channels):
+ *
+ * <-16 weights-> <---------32 0s--------->
+ * X X ...... X X 0 0 0 ............. 0 0 0
+ * 0 .......... 0 X X .... X X 0 ...... 0 0
+ * 0 0 0 ............... 0 0 0 X X .... X X
+ *
+ * To be clear, the second row should have 16 0s followed by 16
+ * weights followed by 16 0s.
+ *
+ * To create the above filter, we take advantage of the fact that
+ * between two sets of non-zero weights, there is always a sequence
+ * of C*K*K 0s. In the above example, C*K^2 = 48 (e.g., 32 trailing
+ * 0s in the first row and 16 leading 0s in the second row).
+ *
+ * Note that, in the special case of C=1 we do not need to
+ * intersperse with 0s (no question of being channel-wise independent
+ * since we have only 1 channel).
+ */
+ #if(C > 1){
+ /*
+ * Append C*K*K trailing 0s to the K*K kernel and replicate the
+ * resulting row C times
+ */
+ repl_weights = matrix(1, rows=C, cols=1) %*% cbind(weights, matrix(0, rows=1, cols=C*K*K))
+
+ /*
+ * The above operation added extra C*K*K trailing 0s in the last row
+ * that we do not need. Thus, we need to:
+ * 1) reshape the resulting matrix into a row
+ * 2) 'Clip off' the last few 0s using indexing and reshape the
+ * result into the expected filter shape ([C, C, K, K])
+ */
+ repl_weights_row = matrix(repl_weights, rows=1, cols=C*(C+1)*K^2)
+ W = matrix(repl_weights_row[1,1:(C*K)^2], rows=C, cols=C*K^2)
+ #}else W = weights
+
+ b = matrix(0, rows=C, cols=1)
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d04d2381/scripts/nn/test/grad_check.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/test/grad_check.dml b/scripts/nn/test/grad_check.dml
index e767eee..5e57081 100644
--- a/scripts/nn/test/grad_check.dml
+++ b/scripts/nn/test/grad_check.dml
@@ -27,6 +27,7 @@ source("nn/layers/batch_norm1d.dml") as batch_norm1d
source("nn/layers/batch_norm2d.dml") as batch_norm2d
source("nn/layers/conv2d.dml") as conv2d
source("nn/layers/conv2d_builtin.dml") as conv2d_builtin
+source("nn/layers/conv2d_transpose.dml") as conv2d_transpose
source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss
source("nn/layers/dropout.dml") as dropout
source("nn/layers/l1_loss.dml") as l1_loss
@@ -616,6 +617,114 @@ conv2d_simple = function() {
}
}
+conv2d_transpose = function() {
+ /*
+ * Gradient check for the 2D convolution transpose layer.
+ */
+ print("Grad checking the 2D convolution transpose layer with L2 loss.")
+
+ N = 2
+ C = 2
+ Hin = 3
+ Win = 3
+ F = 2
+ Hf = 3
+ Wf = 3
+ stride = 2
+ pad = 1
+ out_pad = 1
+
+ X = rand(rows=N, cols=C*Hin*Win)
+
+ [W,b] = conv2d_transpose::init(F, C, Hf, Wf)
+
+ [out, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ y = rand(rows=N, cols=F*Hout*Wout)
+
+ dout = l2_loss::backward(out,y)
+
+ [dX, dW, db] = conv2d_transpose::backward(dout, Hout, Wout, X, W, b, C, Hin, Win, Hf, Wf,
+ stride, stride, pad, pad)
+
+ h = 1e-5
+ print(" - Grad checking X.")
+ for (i in 1:nrow(X)) {
+ for (j in 1:ncol(X)) {
+ old = as.scalar(X[i,j])
+ X[i,j] = old - h
+
+ [outmh, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossmh = l2_loss::forward(outmh, y)
+
+ X[i,j] = old + h
+ [outph, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossph = l2_loss::forward(outph, y)
+
+ X[i,j] = old
+
+ dX_num = (lossph-lossmh) / (2*h)
+
+ rel_error = test_util::check_rel_grad_error(as.scalar(dX[i,j]), dX_num, lossph, lossmh)
+ }
+ }
+
+ print(" - Grad checking W.")
+ for (i in 1:nrow(W)) {
+ for (j in 1:ncol(W)) {
+ old = as.scalar(W[i,j])
+ W[i,j] = old - h
+
+ [outmh, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossmh = l2_loss::forward(outmh, y)
+
+ W[i,j] = old + h
+ [outph, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossph = l2_loss::forward(outph, y)
+
+ W[i,j] = old
+
+ dW_num = (lossph-lossmh) / (2*h)
+
+ rel_error = test_util::check_rel_grad_error(as.scalar(dW[i,j]), dW_num, lossph, lossmh)
+ }
+ }
+
+ print(" - Grad checking b.")
+ for (i in 1:nrow(b)) {
+ for (j in 1:ncol(b)) {
+ old = as.scalar(b[i,j])
+ b[i,j] = old - h
+
+ [outmh, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossmh = l2_loss::forward(outmh, y)
+
+ b[i,j] = old + h
+ [outph, Hout, Wout] = conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride,
+ pad, pad, out_pad, out_pad)
+
+ lossph = l2_loss::forward(outph, y)
+
+ b[i,j] = old
+
+ db_num = (lossph-lossmh) / (2*h)
+
+ rel_error = test_util::check_rel_grad_error(as.scalar(db[i,j]), db_num, lossph, lossmh)
+ }
+ }
+}
+
cross_entropy_loss = function() {
/*
* Gradient check for the cross-entropy loss function.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d04d2381/scripts/nn/test/run_tests.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/test/run_tests.dml b/scripts/nn/test/run_tests.dml
index b061d26..b48606c 100644
--- a/scripts/nn/test/run_tests.dml
+++ b/scripts/nn/test/run_tests.dml
@@ -45,6 +45,7 @@ grad_check::batch_norm2d()
grad_check::conv2d()
grad_check::conv2d_builtin()
grad_check::conv2d_simple()
+grad_check::conv2d_transpose()
grad_check::dropout()
grad_check::lstm()
grad_check::max_pool2d()
@@ -85,6 +86,7 @@ print("---")
test::batch_norm1d()
test::batch_norm2d()
test::conv2d()
+test::conv2d_transpose()
test::cross_entropy_loss()
test::im2col()
test::max_pool2d()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d04d2381/scripts/nn/test/test.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/test/test.dml b/scripts/nn/test/test.dml
index a5cb497..0df7f0f 100644
--- a/scripts/nn/test/test.dml
+++ b/scripts/nn/test/test.dml
@@ -26,6 +26,7 @@ source("nn/layers/batch_norm1d.dml") as batch_norm1d
source("nn/layers/batch_norm2d.dml") as batch_norm2d
source("nn/layers/conv2d.dml") as conv2d
source("nn/layers/conv2d_builtin.dml") as conv2d_builtin
+source("nn/layers/conv2d_transpose.dml") as conv2d_transpose
source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss
source("nn/layers/max_pool2d.dml") as max_pool2d
source("nn/layers/max_pool2d_builtin.dml") as max_pool2d_builtin
@@ -108,6 +109,42 @@ conv2d = function() {
}
}
+conv2d_transpose = function() {
+ /*
+ * Test for the 2D convolution transpose function.
+ */
+ print("Testing the 2D convolution transpose function.")
+
+ N = 1
+ C = 1
+ Hin = 2
+ Win = 2
+ F = 1
+ Hf = 3
+ Wf = 3
+ stride = 1
+ pad = 0
+ out_pad = 0
+
+ X = matrix(seq(1,N*C*Hin*Win), rows=N, cols=C*Hin*Win)
+ W = matrix(seq(1,F*C*Hf*Wf), rows=F, cols=C*Hf*Wf)
+ b = matrix(0, rows=F, cols=1)
+
+ # Forward
+ [out, Hout, Wout] =
+ conv2d_transpose::forward(X, W, b, C, Hin, Win, Hf, Wf, stride, stride, pad, pad, out_pad, out_pad)
+
+ # Equivalency check
+ target = matrix("1 4 7 6 7 23 33 24 19 53 63 42 21 52 59 36", rows=N, cols=C*Hout*Wout)
+
+ for (i in 1:nrow(out)) {
+ for(j in 1:ncol(out)) {
+ rel_error = test_util::check_rel_error(as.scalar(out[1,i]),
+ as.scalar(target[1,i]), 1e-3, 1e-4)
+ }
+ }
+}
+
cross_entropy_loss = function() {
/*
* Test for the cross-entropy loss function.