You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2017/04/29 00:57:19 UTC
[1/2] incubator-systemml git commit: [SYSTEMML-1564] Add a Java test
suite wrapper around `nn` DML test suite
Repository: incubator-systemml
Updated Branches:
refs/heads/master fb5309808 -> 9cdce10d5
[SYSTEMML-1564] Add a Java test suite wrapper around `nn` DML test suite
The `nn` library contains it's own DML test suite for gradient checks
and unit tests that produces "ERROR: ..." messages if any of the
mathematical operations return incorrect results. Note that this has
helped to find mathematical bugs in the library & engine that do not
result in JVM exceptions. This commit simply adds a Java test harness
that wraps these DML tests, parsing the standard out for "ERROR:"
messages. A new `scripts` test package has been created, with the aim
of eventually testing all of our DML algorithms. Additionally, the
ParserWrapper has been updated to look in the `scripts` folder as a
fallback, which allows for easy testing of our actual scripts.
Closes #474.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9cdce10d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9cdce10d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9cdce10d
Branch: refs/heads/master
Commit: 9cdce10d5ec414dfdf1c90afb7eb6170799a3ac2
Parents: 42ebdef
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Fri Apr 28 17:51:23 2017 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Fri Apr 28 17:54:14 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/parser/ParserWrapper.java | 7 +-
.../test/integration/AutomatedTestBase.java | 45 +++++++++--
.../test/integration/scripts/nn/NNTest.java | 81 ++++++++++++++++++++
.../integration/scripts/nn/ZPackageSuite.java | 37 +++++++++
4 files changed, 163 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/main/java/org/apache/sysml/parser/ParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
index ec479e2..2711b6a 100644
--- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
@@ -24,6 +24,8 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
@@ -110,7 +112,10 @@ public abstract class ParserWrapper {
LOG.debug("Looking for the following file in the local file system: " + script);
if( !LocalFileUtils.validateExternalFilename(script, false) )
throw new LanguageException("Invalid (non-trustworthy) local filename.");
- in = new BufferedReader(new FileReader(script));
+ if (Files.exists(Paths.get(script)))
+ in = new BufferedReader(new FileReader(script));
+ else // check in scripts/ directory for file (useful for tests)
+ in = new BufferedReader(new FileReader("scripts/" + script));
}
//core script reading
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index a822ae7..9bf227d 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -20,6 +20,7 @@
package org.apache.sysml.test.integration;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -228,6 +229,8 @@ public abstract class AutomatedTestBase
private String expectedStdOut;
private int iExpectedStdOutState = 0;
+ private String unexpectedStdOut;
+ private int iUnexpectedStdOutState = 0;
private PrintStream originalPrintStreamStd = null;
private String expectedStdErr;
@@ -839,7 +842,7 @@ public abstract class AutomatedTestBase
String testDirectory = config.getTestDirectory();
if (testDirectory != null) {
if (isTargetTestDirectory(testDirectory)) {
- baseDirectory = TEST_DATA_DIR + testDirectory;
+ baseDirectory = TEST_DATA_DIR + testDirectory;
sourceDirectory = SCRIPT_DIR + getSourceDirectory(testDirectory);
}
else {
@@ -1465,6 +1468,7 @@ public abstract class AutomatedTestBase
|| iExpectedStdOutState == 2);
assertTrue("expected String did not occur (stderr): " + expectedStdErr, iExpectedStdErrState == 0
|| iExpectedStdErrState == 2);
+ assertFalse("unexpected String occurred: " + unexpectedStdOut, iUnexpectedStdOutState == 1);
TestUtils.displayAssertionBuffer();
@@ -1495,7 +1499,7 @@ public abstract class AutomatedTestBase
}
/**
- * Enables expection of a line in standard output stream.
+ * Enables detection of expected output of a line in standard output stream.
*
* @param expectedLine
*/
@@ -1510,8 +1514,6 @@ public abstract class AutomatedTestBase
* This class is used to compare the standard output stream against an
* expected string.
*
- *
- *
*/
class ExpectedOutputStream extends OutputStream {
private String line = "";
@@ -1544,8 +1546,6 @@ public abstract class AutomatedTestBase
* This class is used to compare the standard error stream against an
* expected string.
*
- *
- *
*/
class ExpectedErrorStream extends OutputStream {
private String line = "";
@@ -1568,6 +1568,39 @@ public abstract class AutomatedTestBase
}
/**
+ * Enables detection of unexpected output of a line in standard output stream.
+ *
+ * @param unexpectedLine String that should not occur in stdout.
+ */
+ public void setUnexpectedStdOut(String unexpectedLine) {
+ this.unexpectedStdOut = unexpectedLine;
+ originalPrintStreamStd = System.out;
+ System.setOut(new PrintStream(new UnexpectedOutputStream()));
+ }
+
+ /**
+ * This class is used to compare the standard output stream against
+ * an unexpected string.
+ */
+ class UnexpectedOutputStream extends OutputStream {
+ private String line = "";
+
+ @Override
+ public void write(int b) throws IOException {
+ line += String.valueOf((char) b);
+ if (((char) b) == '\n') {
+ /** new line */
+ if (line.contains(unexpectedStdOut)) {
+ iUnexpectedStdOutState = 1; // error!
+ } else {
+ line = ""; // reset buffer
+ }
+ }
+ originalPrintStreamStd.write(b);
+ }
+ }
+
+ /**
* <p>
* Generates a matrix containing easy to debug values in its cells.
* </p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java b/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java
new file mode 100644
index 0000000..d86b707
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.scripts.nn;
+
+import org.apache.spark.sql.SparkSession;
+import org.apache.sysml.api.mlcontext.MLContext;
+import org.apache.sysml.api.mlcontext.Script;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.sysml.api.mlcontext.ScriptFactory.dmlFromFile;
+
+/**
+ * Test the SystemML deep learning library, `nn`.
+ */
+public class NNTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "NNTest";
+ private static final String TEST_DIR = "scripts/";
+ private static final String TEST_SCRIPT = "scripts/nn/test/run_tests.dml";
+ private static final String ERROR_STRING = "ERROR:";
+
+ private static SparkSession spark;
+ private static MLContext ml;
+
+ @BeforeClass
+ public static void setUpClass() {
+ spark = createSystemMLSparkSession("MLContextTest", "local");
+ ml = new MLContext(spark);
+ }
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_DIR, TEST_NAME);
+ getAndLoadTestConfiguration(TEST_NAME);
+ }
+
+ @Test
+ public void testNNLibrary() {
+ Script script = dmlFromFile(TEST_SCRIPT);
+ setUnexpectedStdOut(ERROR_STRING);
+ ml.execute(script);
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ // stop underlying spark context to allow single jvm tests (otherwise the
+ // next test that tries to create a SparkContext would fail)
+ spark.stop();
+ spark = null;
+
+ // clear status mlcontext and spark exec context
+ ml.close();
+ ml = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java
new file mode 100644
index 0000000..145b954
--- /dev/null
+++ b/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.scripts.nn;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/** Group together the tests in this package/related subpackages into a single suite so that the Maven build
+ * won't run two of them at once. Since the DML and PyDML equivalent tests currently share the same directories,
+ * they should not be run in parallel. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ NNTest.class,
+})
+
+
+/** This class is just a holder for the above JUnit annotations. */
+public class ZPackageSuite {
+
+}
[2/2] incubator-systemml git commit: [SYSTEMML-618] Stabilizing the
`two_layer_affine_l2_net` grad check.
Posted by du...@apache.org.
[SYSTEMML-618] Stabilizing the `two_layer_affine_l2_net` grad check.
The example `two_layer_affine_l2_net` network uses a ReLU nonlinearity,
could result in a false-negative in which the test fails due to a kink
being crossed in the ReLU nonlinearity. This occurs when the tests,
f(x-h) and f(x+h), end up on opposite sides of the zero threshold of
max(0, fx). This change aims to stabilize the test, while still
allowing for the usage of the ReLU nonlinearity. Although we have
stabilized it, we will still remove it from automated testing to avoid
false failures. In the future, we can explicitly check for this
scenario and rerun the test automatically. For manual testing, simply
rerun the tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/42ebdef5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/42ebdef5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/42ebdef5
Branch: refs/heads/master
Commit: 42ebdef5d38ad421e2f478eefb8bea535d402309
Parents: fb53098
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Fri Apr 28 17:51:13 2017 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Fri Apr 28 17:54:14 2017 -0700
----------------------------------------------------------------------
scripts/nn/test/grad_check.dml | 5 +++--
scripts/nn/test/run_tests.dml | 10 +++++++++-
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42ebdef5/scripts/nn/test/grad_check.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/test/grad_check.dml b/scripts/nn/test/grad_check.dml
index f3bc9a7..e767eee 100644
--- a/scripts/nn/test/grad_check.dml
+++ b/scripts/nn/test/grad_check.dml
@@ -1612,11 +1612,12 @@ two_layer_affine_l2_net = function() {
M = 10 # number of hidden neurons
[W1, b1] = affine::init(D, M)
[W2, b2] = affine::init(M, yD)
+ W2 = W2 / sqrt(2) # different initialization, since being fed into l2 loss, instead of relu
# Optimize for short "burn-in" time to move to characteristic
# mode of operation and unmask any real issues.
print(" - Burn-in:")
- lr = 0.0001
+ lr = 0.01
decay = 0.99
for(i in 1:5) {
# Compute forward and backward passes of net
@@ -1635,7 +1636,7 @@ two_layer_affine_l2_net = function() {
[pred, loss, dX, dW1, db1, dW2, db2] = two_layer_affine_l2_net_run(X, y, W1, b1, W2, b2)
# Grad check
- h = 1e-5
+ h = 1e-6
print(" - Grad checking X.")
for (i in 1:2) {
for (j in 1:ncol(X)) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42ebdef5/scripts/nn/test/run_tests.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/test/run_tests.dml b/scripts/nn/test/run_tests.dml
index d8173a9..b061d26 100644
--- a/scripts/nn/test/run_tests.dml
+++ b/scripts/nn/test/run_tests.dml
@@ -60,7 +60,15 @@ grad_check::tanh()
print("")
# Example model
-grad_check::two_layer_affine_l2_net()
+# NOTE: This could result in a false-negative in which the test fails
+# due to a kink being crossed in the ReLU nonlinearity. This occurs
+# when the tests, f(x-h) and f(x+h), end up on opposite sides of the
+# zero threshold of max(0, fx). Although we have stabilized it, we
+# will still remove it here to avoid false failures during automated
+# testing. In the future, we can explicitly check for this scenario
+# and rerun the test automatically. For manual testing, simply
+# rerun the tests.
+#grad_check::two_layer_affine_l2_net()
print("")
print("---")