You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/04/22 06:11:57 UTC
[3/4] systemml git commit: [SYSTEMML-2271] Include scalable
decompositions in integration tests
[SYSTEMML-2271] Include scalable decompositions in integration tests
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d368644a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d368644a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d368644a
Branch: refs/heads/master
Commit: d368644a2341ead8d5dcb47b6290fe7c7f833c7a
Parents: 0209edd
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Apr 21 22:17:55 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Apr 21 22:20:11 2018 -0700
----------------------------------------------------------------------
.../runtime/instructions/InstructionUtils.java | 6 +
.../test/integration/AutomatedTestBase.java | 6 +
.../dml/ScalableDecompositionTest.java | 252 +++++++++++++++++++
.../org/apache/sysml/test/utils/TestUtils.java | 17 +-
.../decomp/ScalableDecomposition.dml | 47 ++++
.../integration/applications/ZPackageSuite.java | 2 +-
6 files changed, 319 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
index 3b531dd..588b8e6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java
@@ -87,6 +87,7 @@ import org.apache.sysml.runtime.instructions.gpu.GPUInstruction.GPUINSTRUCTION_T
import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRType;
import org.apache.sysml.runtime.instructions.spark.SPInstruction.SPType;
import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
+import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
import org.apache.sysml.runtime.matrix.operators.AggregateOperator;
import org.apache.sysml.runtime.matrix.operators.AggregateTernaryOperator;
import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
@@ -865,4 +866,9 @@ public class InstructionUtils
|| WeightedUnaryMM.OPCODE.equalsIgnoreCase(opcode) //mapwumm
|| WeightedUnaryMMR.OPCODE.equalsIgnoreCase(opcode); //redwumm
}
+
+ public static AggregateBinaryOperator getMatMultOperator(int k) {
+ AggregateOperator agg = new AggregateOperator(0, Plus.getPlusFnObject());
+ return new AggregateBinaryOperator(Multiply.getMultiplyFnObject(), agg, k);
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 78f8059..48f6428 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -53,6 +53,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.DataConverter;
@@ -504,6 +505,11 @@ public abstract class AutomatedTestBase
return matrix;
}
+ protected double[][] writeInputMatrixWithMTD(String name, MatrixBlock matrix, boolean bIncludeR) {
+ double[][] data = DataConverter.convertToDoubleMatrix(matrix);
+ return writeInputMatrixWithMTD(name, data, bIncludeR);
+ }
+
protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, boolean bIncludeR) {
MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, -1);
return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java
new file mode 100644
index 0000000..607cac0
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.applications.dml;
+
+import java.util.HashMap;
+
+import org.junit.Test;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.MMTSJ.MMTSJType;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class ScalableDecompositionTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "ScalableDecomposition";
+ private final static String TEST_DIR = "applications/decomp/";
+ private final static String TEST_CLASS_DIR =
+ TEST_DIR + ScalableDecompositionTest.class.getSimpleName() + "/";
+
+ private final static int rows = 1362;
+ private final static int cols = 1362;
+ private final static int blen = 200;
+ private final static double eps = 1e-7;
+
+ private enum DecompType {
+ CHOLESKY, LU, QR, SOLVE, INVERSE
+ }
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "C","D","E" }));
+ }
+
+ @Test
+ public void testCholeskyCP() {
+ runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, false, ExecType.CP);
+ }
+
+ @Test
+ public void testCholeskyRewritesCP() {
+ runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, true, ExecType.CP);
+ }
+
+// @Test
+// public void testCholeskySP() {
+// runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, false, ExecType.SPARK);
+// }
+//
+// @Test
+// public void testCholeskyRewritesSP() {
+// runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, true, ExecType.SPARK);
+// }
+
+// @Test
+// public void testLUDecompCP() {
+// runKMeansTest(TEST_NAME1, DecompType.LU, false, ExecType.CP);
+// }
+//
+// @Test
+// public void testLUDecompRewritesCP() {
+// runKMeansTest(TEST_NAME1, DecompType.LU, true, ExecType.CP);
+// }
+
+// @Test
+// public void testLUDecompSP() {
+// runKMeansTest(TEST_NAME1, DecompType.LU, false, ExecType.SPARK);
+// }
+//
+// @Test
+// public void testLUDecompRewritesSP() {
+// runKMeansTest(TEST_NAME1, DecompType.LU, true, ExecType.SPARK);
+// }
+
+// @Test
+// public void testQRDecompCP() {
+// runKMeansTest(TEST_NAME1, DecompType.QR, false, ExecType.CP);
+// }
+//
+// @Test
+// public void testQRDecompRewritesCP() {
+// runKMeansTest(TEST_NAME1, DecompType.QR, true, ExecType.CP);
+// }
+
+// @Test
+// public void testQRDecompSP() {
+// runKMeansTest(TEST_NAME1, DecompType.QR, false, ExecType.SPARK);
+// }
+//
+// @Test
+// public void testQRDecompRewritesSP() {
+// runKMeansTest(TEST_NAME1, DecompType.QR, true, ExecType.SPARK);
+// }
+
+// @Test
+// public void testSolveCP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.CP);
+// }
+//
+// @Test
+// public void testSolveRewritesCP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.CP);
+// }
+
+// @Test
+// public void testSolveSP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.SPARK);
+// }
+//
+// @Test
+// public void testSolveRewritesSP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.SPARK);
+// }
+
+// @Test
+// public void testInverseCP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.CP);
+// }
+//
+// @Test
+// public void testInverseRewritesCP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.CP);
+// }
+
+// @Test
+// public void testInverseSP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.SPARK);
+// }
+//
+// @Test
+// public void testInverseRewritesSP() {
+// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.SPARK);
+// }
+
+
+ private void runKMeansTest(String testname, DecompType type, boolean rewrites, ExecType instType)
+ {
+ boolean oldFlag1 = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ boolean oldFlag2 = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( instType ){
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = rewrites;
+
+ fullDMLScriptName = SCRIPT_DIR + TEST_DIR + testname + ".dml";
+ programArgs = new String[]{"-explain","-args",
+ String.valueOf(type.ordinal()), String.valueOf(blen),
+ input("A"), input("B"), output("C"), output("D"), output("E") };
+
+ switch( type ) {
+ case CHOLESKY: {
+ MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7);
+ MatrixBlock AtA = A.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT);
+ writeInputMatrixWithMTD("A", AtA, false);
+ runTest(true, false, null, -1);
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
+ MatrixBlock C2 = LibCommonsMath.unaryOperations(AtA, "cholesky");
+ TestUtils.compareMatrices(dmlfile, C2, eps);
+ break;
+ }
+ case SOLVE: {
+ MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7);
+ MatrixBlock b = MatrixBlock.randOperations(cols, 1, 1.0, -1, 1, "uniform", 3);
+ MatrixBlock y = A.aggregateBinaryOperations(A, b, new MatrixBlock(), InstructionUtils.getMatMultOperator(1));
+ writeInputMatrixWithMTD("A", A, false);
+ writeInputMatrixWithMTD("B", y, false);
+ runTest(true, false, null, -1);
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
+ MatrixBlock C2 = LibCommonsMath.matrixMatrixOperations(A, b, "solve");
+ TestUtils.compareMatrices(dmlfile, C2, eps);
+ break;
+ }
+ case LU: {
+ MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7);
+ writeInputMatrixWithMTD("A", A, false);
+ runTest(true, false, null, -1);
+ MatrixBlock[] C = LibCommonsMath.multiReturnOperations(A, "lu");
+ String[] outputs = new String[]{"C","D","E"};
+ for(int i=0; i<outputs.length; i++) {
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(outputs[i]);
+ TestUtils.compareMatrices(dmlfile, C[i], eps);
+ }
+ break;
+ }
+ case QR: {
+ MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7);
+ writeInputMatrixWithMTD("A", A, false);
+ runTest(true, false, null, -1);
+ MatrixBlock[] C = LibCommonsMath.multiReturnOperations(A, "qr");
+ String[] outputs = new String[]{"C","D","E"};
+ for(int i=0; i<outputs.length; i++) {
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(outputs[i]);
+ TestUtils.compareMatrices(dmlfile, C[i], eps);
+ }
+ break;
+ }
+ case INVERSE: {
+ MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7);
+ writeInputMatrixWithMTD("A", A, false);
+ runTest(true, false, null, -1);
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
+ MatrixBlock C2 = LibCommonsMath.unaryOperations(A, "inverse");
+ TestUtils.compareMatrices(dmlfile, C2, eps);
+ break;
+ }
+ }
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag1;
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlag2;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index 5357be0..545ae17 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -62,6 +62,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.test.integration.AutomatedTestBase;
@@ -693,22 +694,18 @@ public class TestUtils
assertEquals(expected, actual);
}
- /**
- *
- * @param m1
- * @param m2
- * @param tolerance
- * @param name1
- * @param name2
- * @param ignoreNaN
- * @return
- */
public static boolean compareMatrices(HashMap<CellIndex, Double> m1, HashMap<CellIndex, Double> m2,
double tolerance, String name1, String name2)
{
return compareMatrices(m1, m2, tolerance, name1, name2, false);
}
+ public static void compareMatrices(HashMap<CellIndex, Double> m1, MatrixBlock m2, double tolerance) {
+ double[][] ret1 = convertHashMapToDoubleArray(m1);
+ double[][] ret2 = DataConverter.convertToDoubleMatrix(m2);
+ compareMatrices(ret1, ret2, m2.getNumRows(), m2.getNumColumns(), tolerance);
+ }
+
/**
* Compares two matrices given as HashMaps. The matrix containing more nnz
* is iterated and each cell value compared against the corresponding cell
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/scripts/applications/decomp/ScalableDecomposition.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/applications/decomp/ScalableDecomposition.dml b/src/test/scripts/applications/decomp/ScalableDecomposition.dml
new file mode 100644
index 0000000..009ddf1
--- /dev/null
+++ b/src/test/scripts/applications/decomp/ScalableDecomposition.dml
@@ -0,0 +1,47 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/staging/scalable_linalg/linalg_decomp.dml") as decomp
+
+A = read($3);
+
+if( $1 == 0 ) {
+ C = decomp::Cholesky(A, $2);
+}
+else if( $1 == 1 ) {
+ [C, D, E] = decomp::LU(A, $2);
+}
+else if( $1 == 2 ) {
+ [C, D] = decomp::QR(A, $2);
+}
+else if( $1 == 3 ) {
+ B = read($4);
+ C = decomp::Solve(A, B, $2);
+}
+else if( $1 == 4 ) {
+ C = decomp::Inverse(A, $2);
+}
+
+write(C, $5);
+if( exists(D) )
+ write(D, $6)
+if( exists(E) )
+ write(E, $7)
http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
index f9d1e91..084b5c6 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java
@@ -27,7 +27,6 @@ import org.junit.runners.Suite;
* they should not be run in parallel. */
@RunWith(Suite.class)
@Suite.SuiteClasses({
-
// .applications.dml package
org.apache.sysml.test.integration.applications.dml.ApplyTransformDMLTest.class,
org.apache.sysml.test.integration.applications.dml.ArimaDMLTest.class,
@@ -45,6 +44,7 @@ import org.junit.runners.Suite;
org.apache.sysml.test.integration.applications.dml.NaiveBayesDMLTest.class,
org.apache.sysml.test.integration.applications.dml.NaiveBayesParforDMLTest.class,
org.apache.sysml.test.integration.applications.dml.PageRankDMLTest.class,
+ org.apache.sysml.test.integration.applications.dml.ScalableDecompositionTest.class,
org.apache.sysml.test.integration.applications.dml.WelchTDMLTest.class,
// .applications.pydml package