[systemml] branch master updated: [SYSTEMML-2121] PCA test for codegenalg suite

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

janardhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git


The following commit(s) were added to refs/heads/master by this push:
     new b48653e  [SYSTEMML-2121] PCA test for codegenalg suite
b48653e is described below

commit b48653e9259aa9625eb828d824307f04b04e2b95
Author: Janardhan Pulivarthi <j1...@protonmail.com>
AuthorDate: Sat Mar 30 03:08:49 2019 +0530

    [SYSTEMML-2121] PCA test for codegenalg suite
    
    This patch adds a test case for algorithm test with codegen enabled against an R script.
    - contains a simple test with Dense Rewrite with CP.
    
    Closes #745
---
 .../functions/codegenalg/AlgorithmPCA.java         | 154 +++++++++++++++++++++
 .../scripts/functions/codegenalg/Algorithm_PCA.R   |  87 ++++++++++++
 2 files changed, 241 insertions(+)

diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmPCA.java b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmPCA.java
new file mode 100644
index 0000000..b0344c5
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmPCA.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.codegenalg;
+
+import java.io.File;
+import java.util.HashMap;
+
+import org.junit.Test;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Assert;
+
+public class AlgorithmPCA extends AutomatedTestBase
+{
+	private final static String TEST_NAME1 = "Algorithm_PCA";
+	private final static String TEST_DIR = "functions/codegenalg/";
+	private final static String TEST_CLASS_DIR = TEST_DIR + AlgorithmPCA.class.getSimpleName() + "/";
+	private final static String TEST_CONF_DEFAULT = "SystemML-config-codegen.xml";
+	private final static File TEST_CONF_FILE_DEFAULT = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_DEFAULT);
+	private final static String TEST_CONF_FUSE_ALL = "SystemML-config-codegen-fuse-all.xml";
+	private final static File TEST_CONF_FILE_FUSE_ALL = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF_FUSE_ALL);
+	private final static String TEST_CONF_FUSE_NO_REDUNDANCY = "SystemML-config-codegen-fuse-no-redundancy.xml";
+	private final static File TEST_CONF_FILE_FUSE_NO_REDUNDANCY = new File(SCRIPT_DIR + TEST_DIR,
+			TEST_CONF_FUSE_NO_REDUNDANCY);
+
+	private enum TestType { DEFAULT, FUSE_ALL, FUSE_NO_REDUNDANCY }
+
+	private final static double eps = 1e-5;
+
+	private final static int rows = 3468;
+	private final static int cols1 = 1007;
+	private final static int cols2 = 987;
+
+	private final static double sparsity1 = 0.7; //dense
+	private final static double sparsity2 = 0.1; //sparse
+
+	private TestType currentTestType = TestType.DEFAULT;
+
+	@Override
+	public void setUp() {
+		TestUtils.clearAssertionInformation();
+		addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "w" }));
+	}
+
+	@Test
+	public void testPCADenseRewritesCP() {
+		runPCATest(TEST_NAME1, true, false, ExecType.CP, TestType.DEFAULT);
+	}
+
+	private void runPCATest( String testname, boolean rewrites, boolean sparse, ExecType instType, TestType testType)
+	{
+		boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+		RUNTIME_PLATFORM platformOld = rtplatform;
+		switch( instType ){
+			case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+			case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+			default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+		}
+		currentTestType = testType;
+
+		boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+		if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+			DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+		try
+		{
+			String TEST_NAME = testname;
+			TestConfiguration config = getTestConfiguration(TEST_NAME);
+			loadTestConfiguration(config);
+
+			fullDMLScriptName = "scripts/algorithms/PCA.dml";
+			// pass OFMT=text flag, since readDMLMatrixFromHDFS() uses " " separator, not a "," separator.
+			programArgs = new String[]{ "-explain", "-stats", "-nvargs", "OFMT=TEXT","INPUT="+input("A"),
+					"OUTPUT="+output("")};
+
+			rCmd = getRCmd(inputDir(), expectedDir());
+
+			OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+			//generate actual datasets
+			int cols = (instType==ExecType.SPARK) ? cols2 : cols1;
+			double[][] A = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity1:sparsity1, 714);
+			System.out.println(A);
+			writeInputMatrixWithMTD("A", A, true);
+
+			runTest(true, false, null, -1);
+			runRScript(true);
+
+			//compare matrices
+			HashMap<CellIndex, Double> dmleval = readDMLMatrixFromHDFS("dominant.eigen.values");
+			HashMap<CellIndex, Double> reval   = readRMatrixFromFS("dominant.eigen.values");
+			HashMap<CellIndex, Double> dmlevec = readDMLMatrixFromHDFS("dominant.eigen.vectors");
+			HashMap<CellIndex, Double> revec = readDMLMatrixFromHDFS("dominant.eigen.vectors");
+			HashMap<CellIndex, Double> dmlstd = readDMLMatrixFromHDFS("dominant.eigen.standard.deviations");
+			HashMap<CellIndex, Double> rstd   = readRMatrixFromFS("dominant.eigen.standard.deviations");
+			TestUtils.compareMatrices(dmleval, reval, eps, "Stat-DML", "Stat-R");
+			TestUtils.compareMatrices(dmlevec, revec, eps, "Stat-DML", "Stat-R");
+			TestUtils.compareMatrices(dmlstd, rstd, eps, "Stat-DML", "Stat-R");
+			Assert.assertTrue(heavyHittersContainsSubString("spoof") || heavyHittersContainsSubString("sp_spoof"));
+
+		}
+		finally {
+			rtplatform = platformOld;
+			DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+			OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
+			OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
+			OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
+		}
+	}
+
+	/**
+	 * Override default configuration with custom test configuration to ensure
+	 * scratch space and local temporary directory locations are also updated.
+	 */
+	@Override
+	protected File getConfigTemplateFile() {
+		// Instrumentation in this test's output log to show custom configuration file used for template.
+		String message = "This test case overrides default configuration with ";
+		if(currentTestType == AlgorithmPCA.TestType.FUSE_ALL){
+			System.out.println(message + TEST_CONF_FILE_FUSE_ALL.getPath());
+			return TEST_CONF_FILE_FUSE_ALL;
+		} else if(currentTestType == TestType.FUSE_NO_REDUNDANCY){
+			System.out.println(message + TEST_CONF_FILE_FUSE_NO_REDUNDANCY.getPath());
+			return TEST_CONF_FILE_FUSE_NO_REDUNDANCY;
+		} else {
+			System.out.println(message + TEST_CONF_FILE_DEFAULT.getPath());
+			return TEST_CONF_FILE_DEFAULT;
+		}
+	}
+
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/codegenalg/Algorithm_PCA.R b/src/test/scripts/functions/codegenalg/Algorithm_PCA.R
new file mode 100644
index 0000000..338e6a1
--- /dev/null
+++ b/src/test/scripts/functions/codegenalg/Algorithm_PCA.R
@@ -0,0 +1,87 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+#
+# This script performs Principal Component Analysis (PCA) on the given input data.
+#
+
+args <- commandArgs(TRUE)
+library("Matrix")
+
+A = readMM(paste(args[1], "A.mtx", sep=""));
+K = ncol(A);
+projectData = 0;
+model = "";
+center = 0;
+scale = 0;
+
+
+if (model != "") {
+  # reuse existing model to project data
+} else if (model == "") {
+
+  N = nrow(A);
+  D = ncol(A);
+
+  # 1. perform z-scoring (centering and scaling)
+  if (center == 1) {
+    cm = matrix(1, nrow(A), 1) %*% colMeans(A);
+    A = A - cm
+  }
+  if (scale == 1) {
+    cvars = (colSums(A^2));
+    if (center == 1){
+      #cm = colMeans(A);
+      cvars = (cvars - N*(colMeans(A)^2))/(N-1);
+    }
+    Azscored = A / sqrt(cvars);
+    A = Azscored;
+  }
+
+  # 2. compute co-variance matrix
+  mu = colSums(A)/N;
+  C = (t(A) %*% A)/(N-1) - (N/(N-1))*(mu) %*% t(mu);
+
+  # 3. compute eigen vectors and values
+  R <- eigen(C);
+  evalues = R$values;
+  evectors = R$vectors;
+
+  # 4. make an index of values sorted according to magnitude of evalues
+  decreasing_Idx = order(as.vector(evalues), decreasing=TRUE);
+  diagmat = table(seq(1,D), decreasing_Idx);
+  # 5. sorts eigen values by decreasing order
+  evalues = diagmat %*% evalues;
+  # 6. sorts eigen vectors column-wise in the order of decreasing eigen values
+  evectors = evectors %*% diagmat;
+
+  # 7. select K dominant eigen vectors
+  nvec = ncol(evectors); # Here `nvec=K`
+  eval_dominant = evalues[1:K, 1];
+  evec_dominant = evectors[1:K,];
+
+  # 8. compute the std. deviation of dominant evalues
+  eval_stdev_dominant = sqrt(eval_dominant);
+
+  writeMM(as(eval_stdev_dominant, "CsparseMatrix"), paste(args[2],"dominant.eigen.standard.deviations", sep=""));
+  writeMM(as(eval_dominant, "CsparseMatrix"), paste(args[2], "dominant.eigen.values", sep=""));
+  writeMM(as(evec_dominant, "CsparseMatrix"), paste(args[2],"dominant.eigen.vectors", sep=""));
+}
\ No newline at end of file