You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/01/12 23:41:49 UTC
[2/3] incubator-systemml git commit: [SYSTEMML-217] Add New Unary
Aggregates: Variance & Standard Deviation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColVariancesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColVariancesTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColVariancesTest.java
new file mode 100644
index 0000000..83495ce
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/ColVariancesTest.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+/**
+ * Test the column variances function, "colVars(X)".
+ */
+public class ColVariancesTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "ColVariances";
+ private static final String TEST_DIR = "functions/aggregate/";
+ private static final String TEST_CLASS_DIR =
+ TEST_DIR + ColVariancesTest.class.getSimpleName() + "/";
+ private static final String INPUT_NAME = "X";
+ private static final String OUTPUT_NAME = "colVariances";
+
+ private static final String colVarOp = "uacvar";
+ private static final String varOp = "uavar";
+ private static final int rows = 1234;
+ private static final int cols = 1432;
+ private static final double sparsitySparse = 0.2;
+ private static final double sparsityDense = 0.7;
+ private static final double eps = Math.pow(10, -10);
+
+ private enum Sparsity {EMPTY, SPARSE, DENSE}
+ private enum DataType {MATRIX, ROWVECTOR, COLUMNVECTOR}
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_NAME, config);
+ }
+
+ // Dense matrix w/ rewrites
+ @Test
+ public void testColVariancesDenseMatrixRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseMatrixRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseMatrixRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Dense matrix w/o rewrites
+ @Test
+ public void testColVariancesDenseMatrixNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseMatrixNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseMatrixNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Dense row vector w/ rewrites
+ @Test
+ public void testColVariancesDenseRowVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseRowVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseRowVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Dense row vector w/o rewrites
+ @Test
+ public void testColVariancesDenseRowVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseRowVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseRowVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Dense column vector w/ rewrites
+ @Test
+ public void testColVariancesDenseColVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseColVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseColVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Dense column vector w/o rewrites
+ @Test
+ public void testColVariancesDenseColVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesDenseColVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesDenseColVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ // Sparse matrix w/ rewrites
+ @Test
+ public void testColVariancesSparseMatrixRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseMatrixRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseMatrixRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Sparse matrix w/o rewrites
+ @Test
+ public void testColVariancesSparseMatrixNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseMatrixNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseMatrixNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Sparse row vector w/ rewrites
+ @Test
+ public void testColVariancesSparseRowVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseRowVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseRowVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Sparse row vector w/o rewrites
+ @Test
+ public void testColVariancesSparseRowVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseRowVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseRowVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Sparse column vector w/ rewrites
+ @Test
+ public void testColVariancesSparseColVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseColVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseColVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Sparse column vector w/o rewrites
+ @Test
+ public void testColVariancesSparseColVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesSparseColVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesSparseColVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ // Empty matrix w/ rewrites
+ @Test
+ public void testColVariancesEmptyMatrixRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyMatrixRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyMatrixRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Empty matrix w/o rewrites
+ @Test
+ public void testColVariancesEmptyMatrixNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyMatrixNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyMatrixNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Empty row vector w/ rewrites
+ @Test
+ public void testColVariancesEmptyRowVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyRowVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyRowVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Empty row vector w/o rewrites
+ @Test
+ public void testColVariancesEmptyRowVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyRowVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyRowVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Empty column vector w/ rewrites
+ @Test
+ public void testColVariancesEmptyColVectorRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyColVectorRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyColVectorRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Empty column vector w/o rewrites
+ @Test
+ public void testColVariancesEmptyColVectorNoRewritesCP() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testColVariancesEmptyColVectorNoRewritesSpark() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testColVariancesEmptyColVectorNoRewritesMR() {
+ testColVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ /**
+ * Test the column variances function, "colVars(X)", on
+ * dense/sparse matrices/vectors on the CP/Spark/MR platforms.
+ *
+ * @param testName The name of this test case.
+ * @param sparsity Selection between empty, sparse, and dense data.
+ * @param dataType Selection between a matrix, a row vector, and a
+ * column vector.
+ * @param rewrites Whether or not to employ algebraic rewrites.
+ * @param platform Selection between CP/Spark/MR platforms.
+ */
+ private void testColVariances(String testName, Sparsity sparsity, DataType dataType,
+ boolean rewrites, ExecType platform) {
+ // Configure settings for this test case
+ boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch (platform) {
+ case MR:
+ rtplatform = RUNTIME_PLATFORM.HADOOP;
+ break;
+ case SPARK:
+ rtplatform = RUNTIME_PLATFORM.SPARK;
+ break;
+ default:
+ rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
+ break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if (rtplatform == RUNTIME_PLATFORM.SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ // Create and load test configuration
+ getAndLoadTestConfiguration(testName);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testName + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input(INPUT_NAME), output(OUTPUT_NAME)};
+ fullRScriptName = HOME + testName + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
+
+ // Generate data
+ // - sparsity
+ double sparsityVal;
+ switch (sparsity) {
+ case EMPTY:
+ sparsityVal = 0;
+ break;
+ case SPARSE:
+ sparsityVal = sparsitySparse;
+ break;
+ case DENSE:
+ default:
+ sparsityVal = sparsityDense;
+ }
+ // - size
+ int r;
+ int c;
+ switch (dataType) {
+ case ROWVECTOR:
+ r = 1;
+ c = cols;
+ break;
+ case COLUMNVECTOR:
+ r = rows;
+ c = 1;
+ break;
+ case MATRIX:
+ default:
+ r = rows;
+ c = cols;
+ }
+ // - generation
+ double[][] X = getRandomMatrix(r, c, -1, 1, sparsityVal, 7);
+ writeInputMatrixWithMTD(INPUT_NAME, X, true);
+
+ // Run DML and R scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ // Compare output matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+
+ // On CP and Spark modes, check that the following
+ // rewrites occurred:
+ // - colVars(X), where X is a row vector, should be
+ // rewritten to an empty row vector of zeros.
+ // - colVars(X), where X is a column vector, should be
+ // rewritten to var(X).
+ if (rewrites && (platform == ExecType.SPARK || platform == ExecType.CP)) {
+ String prefix = (platform == ExecType.SPARK) ? Instruction.SP_INST_PREFIX : "";
+ if (dataType == DataType.ROWVECTOR) {
+ String opcode = prefix + colVarOp;
+ boolean rewriteApplied = !Statistics.getCPHeavyHitterOpCodes().contains(opcode);
+ Assert.assertTrue("Rewrite not applied to row vector case.", rewriteApplied);
+
+ } else if (dataType == DataType.COLUMNVECTOR) {
+ String opcode = prefix + varOp;
+ boolean rewriteApplied = Statistics.getCPHeavyHitterOpCodes().contains(opcode);
+ Assert.assertTrue("Rewrite not applied to column vector case.", rewriteApplied);
+ }
+ }
+ }
+ finally {
+ // Reset settings
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowStdDevsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowStdDevsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowStdDevsTest.java
new file mode 100644
index 0000000..e2e50fe
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowStdDevsTest.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+/**
+ * Test the row standard deviations function, "rowSds(X)".
+ */
+public class RowStdDevsTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "RowStdDevs";
+ private static final String TEST_DIR = "functions/aggregate/";
+ private static final String TEST_CLASS_DIR =
+ TEST_DIR + RowStdDevsTest.class.getSimpleName() + "/";
+ private static final String INPUT_NAME = "X";
+ private static final String OUTPUT_NAME = "rowStdDevs";
+
+ private static final int rows = 1234;
+ private static final int cols = 1432;
+ private static final double sparsitySparse = 0.2;
+ private static final double sparsityDense = 0.7;
+ private static final double eps = Math.pow(10, -10);
+
+ private enum Sparsity {EMPTY, SPARSE, DENSE}
+ private enum DataType {MATRIX, ROWVECTOR, COLUMNVECTOR}
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_NAME, config);
+ }
+
+ // Dense matrix
+ @Test
+ public void testRowStdDevsDenseMatrixCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsDenseMatrixSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsDenseMatrixMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Dense row vector
+ @Test
+ public void testRowStdDevsDenseRowVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsDenseRowVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsDenseRowVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Dense column vector
+ @Test
+ public void testRowStdDevsDenseColVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsDenseColVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsDenseColVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Sparse matrix
+ @Test
+ public void testRowStdDevsSparseMatrixCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsSparseMatrixSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsSparseMatrixMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Sparse row vector
+ @Test
+ public void testRowStdDevsSparseRowVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsSparseRowVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsSparseRowVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Sparse column vector
+ @Test
+ public void testRowStdDevsSparseColVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsSparseColVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsSparseColVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Empty matrix
+ @Test
+ public void testRowStdDevsEmptyMatrixCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyMatrixSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyMatrixMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Empty row vector
+ @Test
+ public void testRowStdDevsEmptyRowVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyRowVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyRowVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Empty column vector
+ @Test
+ public void testRowStdDevsEmptyColVectorCP() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyColVectorSpark() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowStdDevsEmptyColVectorMR() {
+ testRowStdDevs(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ /**
+ * Test the row standard deviations function, "rowSds(X)", on
+ * dense/sparse matrices/vectors on the CP/Spark/MR platforms.
+ *
+ * @param testName The name of this test case.
+ * @param sparsity Selection between empty, sparse, and dense data.
+ * @param dataType Selection between a matrix, a row vector, and a
+ * column vector.
+ * @param platform Selection between CP/Spark/MR platforms.
+ */
+ private void testRowStdDevs(String testName, Sparsity sparsity, DataType dataType,
+ ExecType platform) {
+ // Configure settings for this test case
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch (platform) {
+ case MR:
+ rtplatform = RUNTIME_PLATFORM.HADOOP;
+ break;
+ case SPARK:
+ rtplatform = RUNTIME_PLATFORM.SPARK;
+ break;
+ default:
+ rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
+ break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if (rtplatform == RUNTIME_PLATFORM.SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ // Create and load test configuration
+ getAndLoadTestConfiguration(testName);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testName + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input(INPUT_NAME), output(OUTPUT_NAME)};
+ fullRScriptName = HOME + testName + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
+
+ // Generate data
+ // - sparsity
+ double sparsityVal;
+ switch (sparsity) {
+ case EMPTY:
+ sparsityVal = 0;
+ break;
+ case SPARSE:
+ sparsityVal = sparsitySparse;
+ break;
+ case DENSE:
+ default:
+ sparsityVal = sparsityDense;
+ }
+ // - size
+ int r;
+ int c;
+ switch (dataType) {
+ case ROWVECTOR:
+ r = 1;
+ c = cols;
+ break;
+ case COLUMNVECTOR:
+ r = rows;
+ c = 1;
+ break;
+ case MATRIX:
+ default:
+ r = rows;
+ c = cols;
+ }
+ // - generation
+ double[][] X = getRandomMatrix(r, c, -1, 1, sparsityVal, 7);
+ writeInputMatrixWithMTD(INPUT_NAME, X, true);
+
+ // Run DML and R scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ // Compare output matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+ }
+ finally {
+ // Reset settings
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowVariancesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowVariancesTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowVariancesTest.java
new file mode 100644
index 0000000..3557859
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/RowVariancesTest.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+/**
+ * Test the row variances function, "rowVars(X)".
+ */
+public class RowVariancesTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "RowVariances";
+ private static final String TEST_DIR = "functions/aggregate/";
+ private static final String TEST_CLASS_DIR =
+ TEST_DIR + RowVariancesTest.class.getSimpleName() + "/";
+ private static final String INPUT_NAME = "X";
+ private static final String OUTPUT_NAME = "rowVariances";
+
+ private static final String rowVarOp = "uarvar";
+ private static final String varOp = "uavar";
+ private static final int rows = 1234;
+ private static final int cols = 1432;
+ private static final double sparsitySparse = 0.2;
+ private static final double sparsityDense = 0.7;
+ private static final double eps = Math.pow(10, -10);
+
+ private enum Sparsity {EMPTY, SPARSE, DENSE}
+ private enum DataType {MATRIX, ROWVECTOR, COLUMNVECTOR}
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_NAME, config);
+ }
+
+ // Dense matrix w/ rewrites
+ @Test
+ public void testRowVariancesDenseMatrixRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseMatrixRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseMatrixRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Dense matrix w/o rewrites
+ @Test
+ public void testRowVariancesDenseMatrixNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseMatrixNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseMatrixNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Dense row vector w/ rewrites
+ @Test
+ public void testRowVariancesDenseRowVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseRowVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseRowVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Dense row vector w/o rewrites
+ @Test
+ public void testRowVariancesDenseRowVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseRowVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseRowVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Dense column vector w/ rewrites
+ @Test
+ public void testRowVariancesDenseColVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseColVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseColVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Dense column vector w/o rewrites
+ @Test
+ public void testRowVariancesDenseColVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesDenseColVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesDenseColVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ // Sparse matrix w/ rewrites
+ @Test
+ public void testRowVariancesSparseMatrixRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseMatrixRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseMatrixRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Sparse matrix w/o rewrites
+ @Test
+ public void testRowVariancesSparseMatrixNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseMatrixNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseMatrixNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Sparse row vector w/ rewrites
+ @Test
+ public void testRowVariancesSparseRowVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseRowVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseRowVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Sparse row vector w/o rewrites
+ @Test
+ public void testRowVariancesSparseRowVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseRowVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseRowVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Sparse column vector w/ rewrites
+ @Test
+ public void testRowVariancesSparseColVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseColVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseColVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Sparse column vector w/o rewrites
+ @Test
+ public void testRowVariancesSparseColVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesSparseColVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesSparseColVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ // Empty matrix w/ rewrites
+ @Test
+ public void testRowVariancesEmptyMatrixRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyMatrixRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyMatrixRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, true, ExecType.MR);
+ }
+
+ // Empty matrix w/o rewrites
+ @Test
+ public void testRowVariancesEmptyMatrixNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyMatrixNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyMatrixNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, false, ExecType.MR);
+ }
+
+ // Empty row vector w/ rewrites
+ @Test
+ public void testRowVariancesEmptyRowVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyRowVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyRowVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, true, ExecType.MR);
+ }
+
+ // Empty row vector w/o rewrites
+ @Test
+ public void testRowVariancesEmptyRowVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyRowVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyRowVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, false, ExecType.MR);
+ }
+
+ // Empty column vector w/ rewrites
+ @Test
+ public void testRowVariancesEmptyColVectorRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyColVectorRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyColVectorRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, true, ExecType.MR);
+ }
+
+ // Empty column vector w/o rewrites
+ @Test
+ public void testRowVariancesEmptyColVectorNoRewritesCP() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.CP);
+ }
+
+ @Test
+ public void testRowVariancesEmptyColVectorNoRewritesSpark() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testRowVariancesEmptyColVectorNoRewritesMR() {
+ testRowVariances(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, false, ExecType.MR);
+ }
+
+ /**
+ * Test the row variances function, "rowVars(X)", on
+ * dense/sparse matrices/vectors on the CP/Spark/MR platforms.
+ *
+ * @param testName The name of this test case.
+ * @param sparsity Selection between empty, sparse, and dense data.
+ * @param dataType Selection between a matrix, a row vector, and a
+ * column vector.
+ * @param rewrites Whether or not to employ algebraic rewrites.
+ * @param platform Selection between CP/Spark/MR platforms.
+ */
+ private void testRowVariances(String testName, Sparsity sparsity, DataType dataType,
+ boolean rewrites, ExecType platform) {
+ // Configure settings for this test case
+ boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch (platform) {
+ case MR:
+ rtplatform = RUNTIME_PLATFORM.HADOOP;
+ break;
+ case SPARK:
+ rtplatform = RUNTIME_PLATFORM.SPARK;
+ break;
+ default:
+ rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
+ break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if (rtplatform == RUNTIME_PLATFORM.SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ // Create and load test configuration
+ getAndLoadTestConfiguration(testName);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testName + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input(INPUT_NAME), output(OUTPUT_NAME)};
+ fullRScriptName = HOME + testName + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
+
+ // Generate data
+ // - sparsity
+ double sparsityVal;
+ switch (sparsity) {
+ case EMPTY:
+ sparsityVal = 0;
+ break;
+ case SPARSE:
+ sparsityVal = sparsitySparse;
+ break;
+ case DENSE:
+ default:
+ sparsityVal = sparsityDense;
+ }
+ // - size
+ int r;
+ int c;
+ switch (dataType) {
+ case ROWVECTOR:
+ r = 1;
+ c = cols;
+ break;
+ case COLUMNVECTOR:
+ r = rows;
+ c = 1;
+ break;
+ case MATRIX:
+ default:
+ r = rows;
+ c = cols;
+ }
+ // - generation
+ double[][] X = getRandomMatrix(r, c, -1, 1, sparsityVal, 7);
+ writeInputMatrixWithMTD(INPUT_NAME, X, true);
+
+ // Run DML and R scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ // Compare output matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+
+ // On CP and Spark modes, check that the following
+ // rewrites occurred:
+ // - rowVars(X), where X is a row vector, should be
+ // rewritten to var(X).
+ // - rowVars(X), where X is a column vector, should be
+ // rewritten to an empty row vector of zeros.
+ if (rewrites && (platform == ExecType.SPARK || platform == ExecType.CP)) {
+ String prefix = (platform == ExecType.SPARK) ? Instruction.SP_INST_PREFIX : "";
+ if (dataType == DataType.ROWVECTOR) {
+ String opcode = prefix + varOp;
+ boolean rewriteApplied = Statistics.getCPHeavyHitterOpCodes().contains(opcode);
+ Assert.assertTrue("Rewrite not applied to row vector case.", rewriteApplied);
+
+ } else if (dataType == DataType.COLUMNVECTOR) {
+ String opcode = prefix + rowVarOp;
+ boolean rewriteApplied = !Statistics.getCPHeavyHitterOpCodes().contains(opcode);
+ Assert.assertTrue("Rewrite not applied to column vector case.", rewriteApplied);
+ }
+ }
+ }
+ finally {
+ // Reset settings
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld;
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/java/org/apache/sysml/test/integration/functions/aggregate/StdDevTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/StdDevTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/StdDevTest.java
new file mode 100644
index 0000000..fc98a50
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/StdDevTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+/**
+ * Test the standard deviation function, "sd(X)".
+ */
+public class StdDevTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "StdDev";
+ private static final String TEST_DIR = "functions/aggregate/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + StdDevTest.class.getSimpleName() + "/";
+ private static final String INPUT_NAME = "X";
+ private static final String OUTPUT_NAME = "stdDev";
+
+ private static final int rows = 1234;
+ private static final int cols = 1432;
+ private static final double sparsitySparse = 0.2;
+ private static final double sparsityDense = 0.7;
+ private static final double eps = Math.pow(10, -10);
+
+ private enum Sparsity {EMPTY, SPARSE, DENSE}
+ private enum DataType {MATRIX, ROWVECTOR, COLUMNVECTOR}
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_NAME, config);
+ }
+
+ // Dense matrix
+ @Test
+ public void testStdDevDenseMatrixCP() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevDenseMatrixSpark() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevDenseMatrixMR() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Dense row vector
+ @Test
+ public void testStdDevDenseRowVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevDenseRowVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevDenseRowVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Dense column vector
+ @Test
+ public void testStdDevDenseColVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevDenseColVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevDenseColVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Sparse matrix
+ @Test
+ public void testStdDevSparseMatrixCP() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevSparseMatrixSpark() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevSparseMatrixMR() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Sparse row vector
+ @Test
+ public void testStdDevSparseRowVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevSparseRowVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevSparseRowVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Sparse column vector
+ @Test
+ public void testStdDevSparseColVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevSparseColVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevSparseColVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Empty matrix
+ @Test
+ public void testStdDevEmptyMatrixCP() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevEmptyMatrixSpark() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevEmptyMatrixMR() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Empty row vector
+ @Test
+ public void testStdDevEmptyRowVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevEmptyRowVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevEmptyRowVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Empty column vector
+ @Test
+ public void testStdDevEmptyColVectorCP() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testStdDevEmptyColVectorSpark() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStdDevEmptyColVectorMR() {
+ testStdDev(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ /**
+ * Test the standard deviation function, "sd(X)", on
+ * dense/sparse matrices/vectors on the CP/Spark/MR platforms.
+ *
+ * @param testName The name of this test case.
+ * @param sparsity Selection between empty, sparse, and dense data.
+ * @param dataType Selection between a matrix, a row vector, and a
+ * column vector.
+ * @param platform Selection between CP/Spark/MR platforms.
+ */
+ private void testStdDev(String testName, Sparsity sparsity, DataType dataType,
+ ExecType platform) {
+ // Configure settings for this test case
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch (platform) {
+ case MR:
+ rtplatform = RUNTIME_PLATFORM.HADOOP;
+ break;
+ case SPARK:
+ rtplatform = RUNTIME_PLATFORM.SPARK;
+ break;
+ default:
+ rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
+ break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if (rtplatform == RUNTIME_PLATFORM.SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ // Create and load test configuration
+ getAndLoadTestConfiguration(testName);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testName + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input(INPUT_NAME), output(OUTPUT_NAME)};
+ fullRScriptName = HOME + testName + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
+
+ // Generate data
+ // - sparsity
+ double sparsityVal;
+ switch (sparsity) {
+ case EMPTY:
+ sparsityVal = 0;
+ break;
+ case SPARSE:
+ sparsityVal = sparsitySparse;
+ break;
+ case DENSE:
+ default:
+ sparsityVal = sparsityDense;
+ }
+ // - size
+ int r;
+ int c;
+ switch (dataType) {
+ case ROWVECTOR:
+ r = 1;
+ c = cols;
+ break;
+ case COLUMNVECTOR:
+ r = rows;
+ c = 1;
+ break;
+ case MATRIX:
+ default:
+ r = rows;
+ c = cols;
+ }
+ // - generation
+ double[][] X = getRandomMatrix(r, c, -1, 1, sparsityVal, 7);
+ writeInputMatrixWithMTD(INPUT_NAME, X, true);
+
+ // Run DML and R scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ // Compare output matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+ }
+ finally {
+ // Reset settings
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/java/org/apache/sysml/test/integration/functions/aggregate/VarianceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/VarianceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/VarianceTest.java
new file mode 100644
index 0000000..c44bf0d
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/VarianceTest.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.aggregate;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+/**
+ * Test the variance function, "var(X)".
+ */
+public class VarianceTest extends AutomatedTestBase {
+
+ private static final String TEST_NAME = "Variance";
+ private static final String TEST_DIR = "functions/aggregate/";
+ private static final String TEST_CLASS_DIR =
+ TEST_DIR + VarianceTest.class.getSimpleName() + "/";
+ private static final String INPUT_NAME = "X";
+ private static final String OUTPUT_NAME = "variance";
+
+ private static final int rows = 1234;
+ private static final int cols = 1432;
+ private static final double sparsitySparse = 0.2;
+ private static final double sparsityDense = 0.7;
+ private static final double eps = Math.pow(10, -10);
+
+ private enum Sparsity {EMPTY, SPARSE, DENSE}
+ private enum DataType {MATRIX, ROWVECTOR, COLUMNVECTOR}
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_NAME, config);
+ }
+
+ // Dense matrix
+ @Test
+ public void testVarianceDenseMatrixCP() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceDenseMatrixSpark() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceDenseMatrixMR() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Dense row vector
+ @Test
+ public void testVarianceDenseRowVectorCP() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceDenseRowVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceDenseRowVectorMR() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Dense column vector
+ @Test
+ public void testVarianceDenseColVectorCP() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceDenseColVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceDenseColVectorMR() {
+ testVariance(TEST_NAME, Sparsity.DENSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Sparse matrix
+ @Test
+ public void testVarianceSparseMatrixCP() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceSparseMatrixSpark() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceSparseMatrixMR() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Sparse row vector
+ @Test
+ public void testVarianceSparseRowVectorCP() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceSparseRowVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceSparseRowVectorMR() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Sparse column vector
+ @Test
+ public void testVarianceSparseColVectorCP() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceSparseColVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceSparseColVectorMR() {
+ testVariance(TEST_NAME, Sparsity.SPARSE, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ // Empty matrix
+ @Test
+ public void testVarianceEmptyMatrixCP() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceEmptyMatrixSpark() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceEmptyMatrixMR() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.MATRIX, ExecType.MR);
+ }
+
+ // Empty row vector
+ @Test
+ public void testVarianceEmptyRowVectorCP() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceEmptyRowVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceEmptyRowVectorMR() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.ROWVECTOR, ExecType.MR);
+ }
+
+ // Empty column vector
+ @Test
+ public void testVarianceEmptyColVectorCP() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.CP);
+ }
+
+ @Test
+ public void testVarianceEmptyColVectorSpark() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.SPARK);
+ }
+
+ @Test
+ public void testVarianceEmptyColVectorMR() {
+ testVariance(TEST_NAME, Sparsity.EMPTY, DataType.COLUMNVECTOR, ExecType.MR);
+ }
+
+ /**
+ * Test the variance function, "var(X)", on
+ * dense/sparse matrices/vectors on the CP/Spark/MR platforms.
+ *
+ * @param testName The name of this test case.
+ * @param sparsity Selection between empty, sparse, and dense data.
+ * @param dataType Selection between a matrix, a row vector, and a
+ * column vector.
+ * @param platform Selection between CP/Spark/MR platforms.
+ */
+ private void testVariance(String testName, Sparsity sparsity, DataType dataType,
+ ExecType platform) {
+ // Configure settings for this test case
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch (platform) {
+ case MR:
+ rtplatform = RUNTIME_PLATFORM.HADOOP;
+ break;
+ case SPARK:
+ rtplatform = RUNTIME_PLATFORM.SPARK;
+ break;
+ default:
+ rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
+ break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if (rtplatform == RUNTIME_PLATFORM.SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try {
+ // Create and load test configuration
+ getAndLoadTestConfiguration(testName);
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testName + ".dml";
+ programArgs = new String[]{"-explain", "-stats", "-args",
+ input(INPUT_NAME), output(OUTPUT_NAME)};
+ fullRScriptName = HOME + testName + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
+
+ // Generate data
+ // - sparsity
+ double sparsityVal;
+ switch (sparsity) {
+ case EMPTY:
+ sparsityVal = 0;
+ break;
+ case SPARSE:
+ sparsityVal = sparsitySparse;
+ break;
+ case DENSE:
+ default:
+ sparsityVal = sparsityDense;
+ }
+ // - size
+ int r;
+ int c;
+ switch (dataType) {
+ case ROWVECTOR:
+ r = 1;
+ c = cols;
+ break;
+ case COLUMNVECTOR:
+ r = rows;
+ c = 1;
+ break;
+ case MATRIX:
+ default:
+ r = rows;
+ c = cols;
+ }
+ // - generation
+ double[][] X = getRandomMatrix(r, c, -1, 1, sparsityVal, 7);
+ writeInputMatrixWithMTD(INPUT_NAME, X, true);
+
+ // Run DML and R scripts
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ // Compare output matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(OUTPUT_NAME);
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS(OUTPUT_NAME);
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+ }
+ finally {
+ // Reset settings
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/ColStdDevs.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColStdDevs.R b/src/test/scripts/functions/aggregate/ColStdDevs.R
new file mode 100644
index 0000000..9d4c1db
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/ColStdDevs.R
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+colStdDevs <- t(colSds(X))
+
+# R outputs the standard deviation of a single number as NA, whereas
+# a more appropriate value would be 0, so replace all NAs with zeros.
+# This is useful in the case of colStdDevs of a row vector.
+colStdDevs[is.na(colStdDevs)] <- 0
+
+writeMM(as(colStdDevs, "CsparseMatrix"), paste(args[2], "colStdDevs", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/ColStdDevs.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColStdDevs.dml b/src/test/scripts/functions/aggregate/ColStdDevs.dml
new file mode 100644
index 0000000..d7eb561
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/ColStdDevs.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+colStdDevs = colSds(X)
+write(colStdDevs, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/ColVariances.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColVariances.R b/src/test/scripts/functions/aggregate/ColVariances.R
new file mode 100644
index 0000000..970f5f8
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/ColVariances.R
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+colVariances <- t(colVars(X))
+
+# R outputs the variance of a single number as NA, whereas a more
+# appropriate value would be 0, so replace all NAs with zeros.
+# This is useful in the case of colVars of a row vector.
+colVariances[is.na(colVariances)] <- 0
+
+writeMM(as(colVariances, "CsparseMatrix"), paste(args[2], "colVariances", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/ColVariances.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/ColVariances.dml b/src/test/scripts/functions/aggregate/ColVariances.dml
new file mode 100644
index 0000000..1fc2919
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/ColVariances.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+colVariances = colVars(X)
+write(colVariances, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/RowStdDevs.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowStdDevs.R b/src/test/scripts/functions/aggregate/RowStdDevs.R
new file mode 100644
index 0000000..bb2bbf5
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/RowStdDevs.R
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+rowStdDevs <- rowSds(X)
+
+# R outputs the standard deviation of a single number as NA, whereas
+# a more appropriate value would be 0, so replace all NAs with zeros.
+# This is useful in the case of rowStdDevs of a column vector.
+rowStdDevs[is.na(rowStdDevs)] <- 0
+
+writeMM(as(rowStdDevs, "CsparseMatrix"), paste(args[2], "rowStdDevs", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/RowStdDevs.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowStdDevs.dml b/src/test/scripts/functions/aggregate/RowStdDevs.dml
new file mode 100644
index 0000000..f3d0af0
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/RowStdDevs.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+rowStdDevs = rowSds(X)
+write(rowStdDevs, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/RowVariances.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowVariances.R b/src/test/scripts/functions/aggregate/RowVariances.R
new file mode 100644
index 0000000..28405db
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/RowVariances.R
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+library("matrixStats")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+rowVariances <- rowVars(X)
+
+# R outputs the variance of a single number as NA, whereas a more
+# appropriate value would be 0, so replace all NAs with zeros.
+# This is useful in the case of rowVars of a column vector.
+rowVariances[is.na(rowVariances)] <- 0
+
+writeMM(as(rowVariances, "CsparseMatrix"), paste(args[2], "rowVariances", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/RowVariances.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowVariances.dml b/src/test/scripts/functions/aggregate/RowVariances.dml
new file mode 100644
index 0000000..4a52944
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/RowVariances.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+rowVariances = rowVars(X)
+write(rowVariances, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/StdDev.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/StdDev.R b/src/test/scripts/functions/aggregate/StdDev.R
new file mode 100644
index 0000000..2c4f346
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/StdDev.R
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+stdDev <- sd(X)
+
+writeMM(as(stdDev, "CsparseMatrix"), paste(args[2], "stdDev", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/StdDev.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/StdDev.dml b/src/test/scripts/functions/aggregate/StdDev.dml
new file mode 100644
index 0000000..c082c5d
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/StdDev.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+stdDev = as.matrix(sd(X))
+write(stdDev, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/Variance.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/Variance.R b/src/test/scripts/functions/aggregate/Variance.R
new file mode 100644
index 0000000..1442654
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/Variance.R
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X <- as.matrix(readMM(paste(args[1], "X.mtx", sep="")))
+variance <- var(as.vector(X)) # unroll matrix for total variance
+
+writeMM(as(variance, "CsparseMatrix"), paste(args[2], "variance", sep=""));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test/scripts/functions/aggregate/Variance.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/Variance.dml b/src/test/scripts/functions/aggregate/Variance.dml
new file mode 100644
index 0000000..33616e9
--- /dev/null
+++ b/src/test/scripts/functions/aggregate/Variance.dml
@@ -0,0 +1,24 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+variance = as.matrix(var(X))
+write(variance, $2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3206ac14/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
index 39432c5..da5b170 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/aggregate/ZPackageSuite.java
@@ -40,7 +40,13 @@ import org.junit.runners.Suite;
RowSumsSqTest.class,
ColSumsSqTest.class,
TraceTest.class,
-
+ StdDevTest.class,
+ RowStdDevsTest.class,
+ ColStdDevsTest.class,
+ VarianceTest.class,
+ RowVariancesTest.class,
+ ColVariancesTest.class,
+
FullAggregateTest.class,
FullColAggregateTest.class,
FullGroupedAggregateTest.class,