You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@systemml.apache.org by GitBox <gi...@apache.org> on 2019/03/31 10:27:11 UTC
[GitHub] [systemml] j143-bot commented on a change in pull request #861:
[SYSTEMML-2121] AutoEncoder - Add Missing Algorithms to Codegen Test Suite
j143-bot commented on a change in pull request #861: [SYSTEMML-2121] AutoEncoder - Add Missing Algorithms to Codegen Test Suite
URL: https://github.com/apache/systemml/pull/861#discussion_r270659759
##########
File path: src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmAutoEncoder.java
##########
@@ -30,203 +32,247 @@
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
-public class AlgorithmAutoEncoder extends AutomatedTestBase
+public class AlgorithmAutoEncoder extends AutomatedTestBase
{
- private final static String TEST_NAME1 = "Algorithm_AutoEncoder";
- private final static String TEST_DIR = "functions/codegenalg/";
- private final static String TEST_CLASS_DIR = TEST_DIR + AlgorithmAutoEncoder.class.getSimpleName() + "/";
- private final static String TEST_CONF_DEFAULT = "SystemML-config-codegen.xml";
- private final static File TEST_CONF_FILE_DEFAULT = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_DEFAULT);
- private final static String TEST_CONF_FUSE_ALL = "SystemML-config-codegen-fuse-all.xml";
- private final static File TEST_CONF_FILE_FUSE_ALL = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_FUSE_ALL);
- private final static String TEST_CONF_FUSE_NO_REDUNDANCY = "SystemML-config-codegen-fuse-no-redundancy.xml";
- private final static File TEST_CONF_FILE_FUSE_NO_REDUNDANCY = new File(SCRIPT_DIR + TEST_DIR,
- TEST_CONF_FUSE_NO_REDUNDANCY);
-
- private enum TestType { DEFAULT,FUSE_ALL,FUSE_NO_REDUNDANCY }
-
- private final static int rows = 2468;
- private final static int cols = 784;
-
- private final static double sparsity1 = 0.7; //dense
- private final static double sparsity2 = 0.1; //sparse
-
- private final static int H1 = 500;
- private final static int H2 = 2;
- private final static double epochs = 2;
-
- private TestType currentTestType = TestType.DEFAULT;
-
- @Override
- public void setUp() {
- TestUtils.clearAssertionInformation();
- addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "w" }));
- }
-
- //Note: limited cases for SPARK, as lazy evaluation
- //causes very long execution time for this algorithm
-
- @Test
- public void testAutoEncoder256DenseCP() {
- runAutoEncoderTest(256, false, false, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder256DenseRewritesCP() {
- runAutoEncoderTest(256, false, true, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder256SparseCP() {
- runAutoEncoderTest(256, true, false, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder256SparseRewritesCP() {
- runAutoEncoderTest(256, true, true, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512DenseCP() {
- runAutoEncoderTest(512, false, false, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesCP() {
- runAutoEncoderTest(512, false, true, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512SparseCP() {
- runAutoEncoderTest(512, true, false, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesCP() {
- runAutoEncoderTest(512, true, true, ExecType.CP, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder256DenseRewritesSpark() {
- runAutoEncoderTest(256, false, true, ExecType.SPARK, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder256SparseRewritesSpark() {
- runAutoEncoderTest(256, true, true, ExecType.SPARK, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesSpark() {
- runAutoEncoderTest(512, false, true, ExecType.SPARK, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesSpark() {
- runAutoEncoderTest(512, true, true, ExecType.SPARK, TestType.DEFAULT);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesCPFuseAll() {
- runAutoEncoderTest(512, false, true, ExecType.CP, TestType.FUSE_ALL);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesCPFuseAll() {
- runAutoEncoderTest(512, true, true, ExecType.CP, TestType.FUSE_ALL);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesSparkFuseAll() {
- runAutoEncoderTest(512, false, true, ExecType.SPARK, TestType.FUSE_ALL);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesSparkFuseAll() {
- runAutoEncoderTest(512, true, true, ExecType.SPARK, TestType.FUSE_ALL);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesCPFuseNoRedundancy() {
- runAutoEncoderTest(512, false, true, ExecType.CP, TestType.FUSE_NO_REDUNDANCY);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesCPFuseNoRedundancy() {
- runAutoEncoderTest(512, true, true, ExecType.CP, TestType.FUSE_NO_REDUNDANCY);
- }
-
- @Test
- public void testAutoEncoder512DenseRewritesSparkFuseNoRedundancy() {
- runAutoEncoderTest(512, false, true, ExecType.SPARK, TestType.FUSE_NO_REDUNDANCY);
- }
-
- @Test
- public void testAutoEncoder512SparseRewritesSparkFuseNoRedundancy() {
- runAutoEncoderTest(512, true, true, ExecType.SPARK, TestType.FUSE_NO_REDUNDANCY);
- }
-
- private void runAutoEncoderTest(int batchsize, boolean sparse, boolean rewrites, ExecType instType, TestType testType)
- {
- boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
- boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- RUNTIME_PLATFORM platformOld = setRuntimePlatform(instType);
- if(shouldSkipTest())
- return;
-
- currentTestType = testType;
- try
- {
- String TEST_NAME = TEST_NAME1;
- TestConfiguration config = getTestConfiguration(TEST_NAME);
- loadTestConfiguration(config);
-
- fullDMLScriptName = "scripts/staging/autoencoder-2layer.dml";
- programArgs = new String[]{ "-explain", "-stats", "-nvargs", "X="+input("X"),
- "H1="+H1, "H2="+H2, "EPOCH="+epochs, "BATCH="+batchsize,
- "W1_out="+output("W1"), "b1_out="+output("b1"),
- "W2_out="+output("W2"), "b2_out="+output("b2"),
- "W3_out="+output("W3"), "b3_out="+output("b3"),
- "W4_out="+output("W4"), "b4_out="+output("b4")};
- OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
-
- //generate actual datasets
- double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 714);
- writeInputMatrixWithMTD("X", X, true);
-
- //run script
- runTest(true, false, null, -1);
- //TODO R script
-
- assertTrue(heavyHittersContainsSubString("spoof")
- || heavyHittersContainsSubString("sp_spoof"));
- }
- finally {
- rtplatform = platformOld;
- DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
- OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
- OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
- OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
- }
- }
-
- /**
- * Override default configuration with custom test configuration to ensure
- * scratch space and local temporary directory locations are also updated.
- */
- @Override
- protected File getConfigTemplateFile() {
- // Instrumentation in this test's output log to show custom configuration file used for template.
- String message = "This test case overrides default configuration with ";
- if(currentTestType == TestType.FUSE_ALL){
- System.out.println(message + TEST_CONF_FILE_FUSE_ALL.getPath());
- return TEST_CONF_FILE_FUSE_ALL;
- } else if(currentTestType == TestType.FUSE_NO_REDUNDANCY){
- System.out.println(message + TEST_CONF_FILE_FUSE_NO_REDUNDANCY.getPath());
- return TEST_CONF_FILE_FUSE_NO_REDUNDANCY;
- } else {
- System.out.println(message + TEST_CONF_FILE_DEFAULT.getPath());
- return TEST_CONF_FILE_DEFAULT;
- }
- }
+ private final static String TEST_NAME1 = "Algorithm_AutoEncoder";
+ private final static String TEST_DIR = "functions/codegenalg/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + AlgorithmAutoEncoder.class.getSimpleName() + "/";
+ private final static String TEST_CONF_DEFAULT = "SystemML-config-codegen.xml";
+ private final static File TEST_CONF_FILE_DEFAULT = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_DEFAULT);
+ private final static String TEST_CONF_FUSE_ALL = "SystemML-config-codegen-fuse-all.xml";
+ private final static File TEST_CONF_FILE_FUSE_ALL = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_FUSE_ALL);
+ private final static String TEST_CONF_FUSE_NO_REDUNDANCY = "SystemML-config-codegen-fuse-no-redundancy.xml";
+ private final static File TEST_CONF_FILE_FUSE_NO_REDUNDANCY = new File(SCRIPT_DIR + TEST_DIR,
+ TEST_CONF_FUSE_NO_REDUNDANCY);
+
+ private enum TestType { DEFAULT,FUSE_ALL,FUSE_NO_REDUNDANCY }
+
+ private final static int rows = 2468;//2468;
+ private final static int cols = 784;
+
+ private final static double sparsity1 = 0.7; //dense
+ private final static double sparsity2 = 0.1; //sparse
+ private final static double eps = 1e-5;
+
+ private final static int H1 = 500;
+ private final static int H2 = 2;
+ private final static double epochs = 100;
+
+ private TestType currentTestType = TestType.DEFAULT;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "w" }));
+ }
+
+ //Note: limited cases for SPARK, as lazy evaluation
+ //causes very long execution time for this algorithm
+
+ @Test
+ public void testAutoEncoder256DenseCP() {
+ runAutoEncoderTest(256, false, false, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder256DenseRewritesCP() {
+ runAutoEncoderTest(256, false, true, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder256SparseCP() {
+ runAutoEncoderTest(256, true, false, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder256SparseRewritesCP() {
+ runAutoEncoderTest(256, true, true, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseCP() {
+ runAutoEncoderTest(512, false, false, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseRewritesCP() {
+ runAutoEncoderTest(512, false, true, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512SparseCP() {
+ runAutoEncoderTest(512, true, false, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512SparseRewritesCP() {
+ runAutoEncoderTest(512, true, true, ExecType.CP, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder256DenseRewritesSpark() {
+ runAutoEncoderTest(256, false, true, ExecType.SPARK, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder256SparseRewritesSpark() {
+ runAutoEncoderTest(256, true, true, ExecType.SPARK, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseRewritesSpark() {
+ runAutoEncoderTest(512, false, true, ExecType.SPARK, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512SparseRewritesSpark() {
+ runAutoEncoderTest(512, true, true, ExecType.SPARK, TestType.DEFAULT);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseRewritesCPFuseAll() {
+ runAutoEncoderTest(512, false, true, ExecType.CP, TestType.FUSE_ALL);
+ }
+
+ @Test
+ public void testAutoEncoder512SparseRewritesCPFuseAll() {
+ runAutoEncoderTest(512, true, true, ExecType.CP, TestType.FUSE_ALL);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseRewritesSparkFuseAll() {
+ runAutoEncoderTest(512, false, true, ExecType.SPARK, TestType.FUSE_ALL);
+ }
+
+ @Test
+ public void testAutoEncoder512SparseRewritesSparkFuseAll() {
+ runAutoEncoderTest(512, true, true, ExecType.SPARK, TestType.FUSE_ALL);
+ }
+
+ @Test
+ public void testAutoEncoder512DenseRewritesCPFuseNoRedundancy() {
Review comment:
I'll try to fix the formatting before committing. :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services