You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/09/12 00:39:30 UTC
incubator-systemml git commit: [SYSTEMML-908][SYSTEMML-618] Adding
and improving tests, improving tanh formulation, and updating comments.
Repository: incubator-systemml
Updated Branches:
refs/heads/master bf4669232 -> 01e622f54
[SYSTEMML-908][SYSTEMML-618] Adding and improving tests, improving tanh formulation, and updating comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/01e622f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/01e622f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/01e622f5
Branch: refs/heads/master
Commit: 01e622f544f92ff87f7f78ad1ce4b5051603afe3
Parents: bf46692
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Sun Sep 11 17:38:52 2016 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Sun Sep 11 17:38:52 2016 -0700
----------------------------------------------------------------------
scripts/staging/SystemML-NN/examples/README.md | 2 +-
.../staging/SystemML-NN/nn/layers/softmax.dml | 2 +-
scripts/staging/SystemML-NN/nn/layers/tanh.dml | 10 ++++--
scripts/staging/SystemML-NN/nn/test/README.md | 32 ++++++++++++++++++++
.../staging/SystemML-NN/nn/test/grad_check.dml | 28 +++++++++--------
scripts/staging/SystemML-NN/nn/test/test.dml | 23 ++++++++++++++
scripts/staging/SystemML-NN/nn/test/tests.dml | 1 +
scripts/staging/SystemML-NN/nn/util.dml | 6 ++--
8 files changed, 83 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/examples/README.md
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/examples/README.md b/scripts/staging/SystemML-NN/examples/README.md
index b1ccf22..ffacea2 100644
--- a/scripts/staging/SystemML-NN/examples/README.md
+++ b/scripts/staging/SystemML-NN/examples/README.md
@@ -67,7 +67,7 @@ limitations under the License.
* **Notebooks**: To run the notebook examples, please install the SystemML Python package with `pip install systemml`, and then startup Jupyter in the following manner from this directory (or for more information, please see [this great blog post](http://spark.tc/0-to-life-changing-application-with-apache-systemml/)):
```
- PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark --master local[*] --driver-memory 3G --driver-class-path $SYSTEMML_HOME/SystemML.jar --jars $SYSTEMML_HOME/SystemML.jar
+ PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark --master local[*] --driver-memory 3G --driver-class-path SystemML.jar --jars SystemML.jar
```
Note that all printed output, such as training statistics, from the SystemML scripts will be sent to the terminal in which Jupyter was started (for now...).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/layers/softmax.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/layers/softmax.dml b/scripts/staging/SystemML-NN/nn/layers/softmax.dml
index 111e1b3..64f257b 100644
--- a/scripts/staging/SystemML-NN/nn/layers/softmax.dml
+++ b/scripts/staging/SystemML-NN/nn/layers/softmax.dml
@@ -32,7 +32,7 @@ forward = function(matrix[double] scores) return (matrix[double] probs) {
* This can be interpreted as a generalization of the sigmoid
* function to multiple classes.
*
- * probs_ij = e^scores_ij / sum(e^scores)
+ * probs_ij = e^scores_ij / sum(e^scores_i)
*
* Inputs:
* - scores: Input data matrix, of shape (N, D).
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/layers/tanh.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/layers/tanh.dml b/scripts/staging/SystemML-NN/nn/layers/tanh.dml
index 0fadf77..9308a7c 100644
--- a/scripts/staging/SystemML-NN/nn/layers/tanh.dml
+++ b/scripts/staging/SystemML-NN/nn/layers/tanh.dml
@@ -22,11 +22,14 @@
/*
* Tanh nonlinearity layer.
*/
+source("nn/layers/sigmoid.dml") as sigmoid
+
forward = function(matrix[double] X) return (matrix[double] out) {
/*
* Computes the forward pass for a tanh nonlinearity layer.
*
- * tanh(x) = (e^x - e^-x) / (e^x + e^-x) = sigmoid(-2x)
+ * tanh(x) = (e^x - e^-x) / (e^x + e^-x)
+ * = 2 * sigmoid(2x) - 1
*
* Inputs:
* - X: Input data matrix, of shape (any, any).
@@ -34,8 +37,9 @@ forward = function(matrix[double] X) return (matrix[double] out) {
* Outputs:
* - out: Ouptuts, of same shape as X.
*/
+ # out = (exp(X) - exp(-X)) / (exp(X) + exp(-X))
# Simplification of the above formulation to use the sigmoid function:
- sigma2X = 1 / (1 + exp(-2*X))
+ sigma2X = sigmoid::forward(2*X)
out = 2 * sigma2X - 1
}
@@ -50,7 +54,7 @@ backward = function(matrix[double] dout, matrix[double] X) return (matrix[double
* Outputs:
* - dX: Gradient wrt X, of same shape as X.
*/
- sigma2X = 1 / (1 + exp(-2*X))
+ sigma2X = sigmoid::forward(2*X)
out = 2 * sigma2X - 1
dX = (1 - out^2) * dout
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/test/README.md
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/test/README.md b/scripts/staging/SystemML-NN/nn/test/README.md
new file mode 100644
index 0000000..b714d50
--- /dev/null
+++ b/scripts/staging/SystemML-NN/nn/test/README.md
@@ -0,0 +1,32 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# SystemML-NN Tests
+
+#### This folder contains tests for the *SystemML-NN* (`nn`) deep learning library.
+
+---
+## Tests
+#### All layers are tested for correct derivatives ("gradient-checking"), and many layers also have correctness tests against simpler reference implementations.
+* `grad_check.dml` - Contains gradient-checks for all layers as individual DML functions.
+* `test.dml` - Contains correctness tests for several of the more complicated layers by checking against simple reference implementations, such as `conv_simple.dml`. All tests are formulated as individual DML functions.
+* `tests.dml` - A DML script that runs all of the tests in `grad_check.dml` and `test.dml`.
+
+## Execution
+* `spark-submit SystemML.jar -f nn/test/tests.dml` from the base of the project.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/test/grad_check.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/test/grad_check.dml b/scripts/staging/SystemML-NN/nn/test/grad_check.dml
index 67bd854..5500c4f 100644
--- a/scripts/staging/SystemML-NN/nn/test/grad_check.dml
+++ b/scripts/staging/SystemML-NN/nn/test/grad_check.dml
@@ -68,15 +68,17 @@ check_rel_error = function(double dw_a, double dw_n, double lossph, double lossm
rel_error = util::compute_rel_error(dw_a, dw_n)
# Evaluate relative error
- if (rel_error > 1e-2) {
- print("ERROR: Relative error " + rel_error + " > 1e-2 with " + dw_a +
- " analytical vs " + dw_n + " numerical, with lossph " + lossph +
- " and lossmh " + lossmh)
+ thresh_error = 1e-2
+ thresh_warn = 1e-4
+ if (rel_error > thresh_error) {
+ print("ERROR: Relative error " + rel_error + " > " + thresh_error + " with " + dw_a +
+ " analytical vs " + dw_n + " numerical, with lossph " + lossph +
+ " and lossmh " + lossmh)
}
- else if (rel_error > 1e-4 & rel_error <= 1e-2) {
- print("WARNING: Relative error " + rel_error + " <= 1e-2 & > 1e-4 with " + dw_a +
- " analytical vs " + dw_n + " numerical, with lossph " + lossph +
- " and lossmh " + lossmh)
+ else if (rel_error > thresh_warn & rel_error <= thresh_error) {
+ print("WARNING: Relative error " + rel_error + " > " + thresh_warn + " & <= " + thresh_error +
+ " with " + dw_a + " analytical vs " + dw_n + " numerical, with lossph " + lossph +
+ " and lossmh " + lossmh)
}
}
@@ -1222,7 +1224,7 @@ two_layer_affine_l2_net = function() {
N = 1000 # num examples
D = 100 # num features
yD = 5 # num targets
- X = rand(rows=N, cols=D, pdf="normal") * 0.0001
+ X = rand(rows=N, cols=D, pdf="normal")
y = rand(rows=N, cols=yD)
# Create 2-layer, fully-connected network
@@ -1257,12 +1259,12 @@ two_layer_affine_l2_net = function() {
for (i in 1:2) {
for (j in 1:ncol(X)) {
# Compute numerical derivative
- old_w = as.scalar(X[i,j])
- X[i,j] = old_w - h
+ old_x = as.scalar(X[i,j])
+ X[i,j] = old_x - h
[lossmh, pred, aout, hout] = two_layer_affine_l2_net_forward(X, y, W1, b1, W2, b2)
- X[i,j] = old_w + h
+ X[i,j] = old_x + h
[lossph, pred, aout, hout] = two_layer_affine_l2_net_forward(X, y, W1, b1, W2, b2)
- X[i,j] = old_w # reset W[i,j]
+ X[i,j] = old_x # reset X[i,j]
dX_num = (lossph - lossmh) / (2 * h) # numerical derivative
# Check error
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/test/test.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/test/test.dml b/scripts/staging/SystemML-NN/nn/test/test.dml
index 1ecff68..d0e83f5 100644
--- a/scripts/staging/SystemML-NN/nn/test/test.dml
+++ b/scripts/staging/SystemML-NN/nn/test/test.dml
@@ -27,6 +27,7 @@ source("nn/layers/conv_builtin.dml") as conv_builtin
source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss
source("nn/layers/max_pool.dml") as max_pool
source("nn/layers/max_pool_builtin.dml") as max_pool_builtin
+source("nn/layers/tanh.dml") as tanh
source("nn/test/conv_simple.dml") as conv_simple
source("nn/test/max_pool_simple.dml") as max_pool_simple
source("nn/util.dml") as util
@@ -218,3 +219,25 @@ max_pool = function() {
tmp = util::check_all_equal(out, target)
}
+tanh = function() {
+ /*
+ * Test for the `tanh` forward function.
+ */
+ print("Testing the tanh forward function.")
+
+ # Generate data
+ N = 2 # num examples
+ C = 3 # num channels
+ X = rand(rows=N, cols=C, pdf="normal")
+
+ out = tanh::forward(X)
+ out_ref = (exp(X) - exp(-X)) / (exp(X) + exp(-X))
+
+ # Equivalency check
+ for (i in 1:nrow(out)) {
+ for (j in 1:ncol(out)) {
+ rel_error = util::check_rel_error(as.scalar(out[i,j]), as.scalar(out_ref[i,j]), 1e-10, 1e-12)
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/test/tests.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/test/tests.dml b/scripts/staging/SystemML-NN/nn/test/tests.dml
index 5535c85..fd897ef 100644
--- a/scripts/staging/SystemML-NN/nn/test/tests.dml
+++ b/scripts/staging/SystemML-NN/nn/test/tests.dml
@@ -67,6 +67,7 @@ tmp = test::padding()
tmp = test::conv()
tmp = test::cross_entropy_loss()
tmp = test::max_pool()
+tmp = test::tanh()
print("---")
print("Other tests complete -- look for any ERRORs or WARNINGs.")
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/01e622f5/scripts/staging/SystemML-NN/nn/util.dml
----------------------------------------------------------------------
diff --git a/scripts/staging/SystemML-NN/nn/util.dml b/scripts/staging/SystemML-NN/nn/util.dml
index 8a97e81..38fddc3 100644
--- a/scripts/staging/SystemML-NN/nn/util.dml
+++ b/scripts/staging/SystemML-NN/nn/util.dml
@@ -105,9 +105,9 @@ check_rel_error = function(double x1, double x2, double thresh_error, double thr
print("ERROR: Relative error " + rel_error + " > " + thresh_error + " with " + x1 +
" vs " + x2 + ".")
}
- else if (rel_error > thresh_warn & rel_error < thresh_error) {
- print("WARNING: Relative error " + rel_error + " > " + thresh_warn + " with " + x1 +
- " vs " + x2 + ".")
+ else if (rel_error > thresh_warn & rel_error <= thresh_error) {
+ print("WARNING: Relative error " + rel_error + " > " + thresh_warn + " & <= " + thresh_error +
+ " with " + x1 + " vs " + x2 + ".")
}
}