You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/12/24 15:15:17 UTC
systemml git commit: [MINOR] New tests for cholesky builtin function,
incl. related fixes
Repository: systemml
Updated Branches:
refs/heads/master 8d8004121 -> 581077712
[MINOR] New tests for cholesky builtin function, incl. related fixes
Closes #710.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/58107771
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/58107771
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/58107771
Branch: refs/heads/master
Commit: 581077712cb1749db67859d4c8a9332ce95c5384
Parents: 8d80041
Author: Janardhan Pulivarthi <j1...@protonmail.com>
Authored: Sun Dec 24 06:13:53 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Dec 24 07:10:33 2017 -0800
----------------------------------------------------------------------
.../IPAPassRemoveUnnecessaryCheckpoints.java | 13 +-
.../cp/BuiltinUnaryCPInstruction.java | 15 +--
.../cp/MatrixBuiltinCPInstruction.java | 2 +-
.../runtime/matrix/data/LibCommonsMath.java | 3 +-
.../functions/unary/matrix/CholeskyTest.java | 128 +++++++++++++++++++
.../scripts/functions/unary/matrix/cholesky.dml | 50 ++++++++
.../functions/unary/matrix/ZPackageSuite.java | 1 +
7 files changed, 195 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java b/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java
index d90b4b6..11bce2f 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java
@@ -214,15 +214,14 @@ public class IPAPassRemoveUnnecessaryCheckpoints extends IPAPass
throws HopsException
{
List<StatementBlock> sbs = dmlp.getStatementBlocks();
-
- if( sbs.size()==1 & !(sbs.get(0) instanceof IfStatementBlock
- || sbs.get(0) instanceof WhileStatementBlock
- || sbs.get(0) instanceof ForStatementBlock) )
- {
+
+ if (sbs.size() == 1 && !(sbs.get(0) instanceof IfStatementBlock
+ || sbs.get(0) instanceof WhileStatementBlock
+ || sbs.get(0) instanceof ForStatementBlock)) {
//recursively process all dag roots
- if( sbs.get(0).getHops()!=null ) {
+ if (sbs.get(0).getHops() != null) {
Hop.resetVisitStatus(sbs.get(0).getHops());
- for( Hop root : sbs.get(0).getHops() )
+ for (Hop root : sbs.get(0).getHops())
rRemoveCheckpointReadWrite(root);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
index 6b055a4..9f72d62 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java
@@ -27,19 +27,19 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.functionobjects.Builtin;
import org.apache.sysml.runtime.functionobjects.ValueFunction;
import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.LibCommonsMath;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction {
- protected BuiltinUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, String opcode,
- String istr) {
+ protected BuiltinUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, String istr) {
super(CPType.BuiltinUnary, op, in, out, opcode, istr);
}
public static BuiltinUnaryCPInstruction parseInstruction ( String str )
- throws DMLRuntimeException
+ throws DMLRuntimeException
{
CPOperand in = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
@@ -49,8 +49,7 @@ public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction {
ValueFunction func = null;
//print or stop or cumulative aggregates
- if( parts.length==4 )
- {
+ if( parts.length==4 ) {
opcode = parts[0];
in.split(parts[1]);
out.split(parts[2]);
@@ -61,15 +60,15 @@ public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction {
else
return new ScalarBuiltinCPInstruction(new SimpleOperator(func), in, out, opcode, str);
}
- else //2+1, general case
- {
+ else { //2+1, general case
opcode = parseUnaryInstruction(str, in, out);
func = Builtin.getBuiltinFnObject(opcode);
if(in.getDataType() == DataType.SCALAR)
return new ScalarBuiltinCPInstruction(new SimpleOperator(func), in, out, opcode, str);
else if(in.getDataType() == DataType.MATRIX)
- return new MatrixBuiltinCPInstruction(new UnaryOperator(func), in, out, opcode, str);
+ return new MatrixBuiltinCPInstruction(LibCommonsMath.isSupportedUnaryOperation(opcode) ?
+ null : new UnaryOperator(func), in, out, opcode, str);
}
return null;
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
index 84e3ebb..b120dab 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java
@@ -35,7 +35,6 @@ public class MatrixBuiltinCPInstruction extends BuiltinUnaryCPInstruction {
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException
{
- UnaryOperator u_op = (UnaryOperator) _optr;
String output_name = output.getName();
String opcode = getOpcode();
@@ -44,6 +43,7 @@ public class MatrixBuiltinCPInstruction extends BuiltinUnaryCPInstruction {
ec.setMatrixOutput(output_name, retBlock, getExtendedOpcode());
}
else {
+ UnaryOperator u_op = (UnaryOperator) _optr;
MatrixBlock inBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
MatrixBlock retBlock = (MatrixBlock) (inBlock.unaryOperations(u_op, new MatrixBlock()));
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
index 30aec56..2755c28 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java
@@ -281,7 +281,8 @@ public class LibCommonsMath
if ( !in.isSquare() )
throw new DMLRuntimeException("Input to cholesky() must be square matrix -- given: a " + in.getRowDimension() + "x" + in.getColumnDimension() + " matrix.");
- CholeskyDecomposition cholesky = new CholeskyDecomposition(in);
+ CholeskyDecomposition cholesky = new CholeskyDecomposition(in, 1e-14,
+ CholeskyDecomposition.DEFAULT_ABSOLUTE_POSITIVITY_THRESHOLD);
RealMatrix rmL = cholesky.getL();
return DataConverter.convertToMatrixBlock(rmL.getData());
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java
new file mode 100644
index 0000000..71b50d6
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.unary.matrix;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the Cholesky matrix factorization
+ * Note:
+ * 1. Only tested for dense configuration of matrices (small, & large)
+ * 2. The remaining tests for matrix dim check, positive definiteness
+ * already tested at commons-math.
+ */
+
+public class CholeskyTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "cholesky";
+ private final static String TEST_DIR = "functions/unary/matrix/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + CholeskyTest.class.getSimpleName() + "/";
+
+ private final static int rows1 = 500;
+ private final static int rows2 = 2500;
+ private final static int cols1 = 500;
+ private final static int cols2 = 2500;
+ private final static double sparsity = 0.9;
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(
+ TEST_NAME,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,
+ new String[] { "D" }) );
+ }
+
+ @Test
+ public void testCholeskyDenseCP() {
+ runTestCholesky( rows1, cols1, DMLScript.RUNTIME_PLATFORM.SINGLE_NODE );
+ }
+
+ @Test
+ public void testCholeskyDenseSP() {
+ runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.SPARK );
+ }
+
+ @Test
+ public void testCholeskyDenseMR() {
+ runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.HADOOP );
+ }
+
+ @Test
+ public void testCholeskyDenseHybrid() {
+ runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.HYBRID );
+ }
+
+ @Test
+ public void testLargeCholeskyDenseCP() {
+ runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.SINGLE_NODE );
+ }
+
+ @Test
+ public void testLargeCholeskyDenseSP() {
+ runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.SPARK );
+ }
+
+ @Test
+ public void testLargeCholeskyDenseMR() {
+ runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.HADOOP );
+ }
+
+ @Test
+ public void testLargeCholeskyDenseHybrid() {
+ runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.HYBRID );
+ }
+
+ private void runTestCholesky( int rows, int cols, RUNTIME_PLATFORM rt) {
+ RUNTIME_PLATFORM rtold = rtplatform;
+ rtplatform = rt;
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ getAndLoadTestConfiguration(TEST_NAME);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME+ TEST_NAME + ".dml";
+ programArgs = new String[]{"-args", input("A"), output("D") };
+
+ double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 10);
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, -1, -1, -1);
+ writeInputMatrixWithMTD("A", A, false, mc);
+
+ //run tests and compare results
+ runTest(true, false, null, -1);
+ Assert.assertEquals(0, readDMLMatrixFromHDFS("D")
+ .get(new CellIndex(1,1)), 1e-5);
+ }
+ finally {
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ rtplatform = rtold;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test/scripts/functions/unary/matrix/cholesky.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/unary/matrix/cholesky.dml b/src/test/scripts/functions/unary/matrix/cholesky.dml
new file mode 100644
index 0000000..8be1e23
--- /dev/null
+++ b/src/test/scripts/functions/unary/matrix/cholesky.dml
@@ -0,0 +1,50 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/*
+ * DML script to test Cholesky Factorization (cholesky), as a Builtin function.
+ *
+ * Reference:
+ * Theorem 4.2.7, page No. 163
+ * "Matrix Computations" by Golub & Van Loan (4th Edition)
+ *
+ * Cholesky Factorization:
+ * If A belongs to R(nxn), is symmetric positive definite,
+ * then there exists a unique lower triangular G belongs to R(nxn), with positive
+ * diagonal entries such that.
+ * A = G %*% t(G);
+ *
+ */
+
+
+X = read($1);
+
+#1. generate a symmetric positive definite matrix
+A = X %*% t(X);
+
+#2. obtain unique lower triangular matrix, G
+G = cholesky(A);
+
+#3. Recompute the positive definite matrix
+B = G %*% t(G);
+D = as.matrix(sum(A-B));
+
+write(D, $2);
http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
index c556138..09f4113 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java
@@ -32,6 +32,7 @@ import org.junit.runners.Suite;
ATanTest.class,
CastAsScalarTest.class,
CosTest.class,
+ CholeskyTest.class,
DiagTest.class,
EigenFactorizeTest.class,
FullCummaxTest.class,