You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/07/20 19:40:19 UTC
systemml git commit: [SYSTEMML-1789] MLContext Univariate Statistics
test
Repository: systemml
Updated Branches:
refs/heads/master cca4f942c -> 3d4c21b01
[SYSTEMML-1789] MLContext Univariate Statistics test
Create MLContext test class for Univar-Stats.dml top-level algorithm.
Closes #585.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3d4c21b0
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3d4c21b0
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3d4c21b0
Branch: refs/heads/master
Commit: 3d4c21b01a891cdae5b2dfc712b2ad7eac82afe6
Parents: cca4f94
Author: Deron Eriksson <de...@apache.org>
Authored: Thu Jul 20 12:37:37 2017 -0700
Committer: Deron Eriksson <de...@apache.org>
Committed: Thu Jul 20 12:37:37 2017 -0700
----------------------------------------------------------------------
.../test/integration/AutomatedTestBase.java | 9 ++
.../MLContextUnivariateStatisticsTest.java | 92 ++++++++++++++++++++
.../mlcontext/algorithms/ZPackageSuite.java | 34 ++++++++
3 files changed, 135 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/3d4c21b0/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 7b93211..5a1b904 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -55,6 +55,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.runtime.util.MapReduceTool;
import org.apache.sysml.test.utils.TestUtils;
import org.apache.sysml.utils.ParameterBuilder;
@@ -1852,4 +1853,12 @@ public abstract class AutomatedTestBase
SparkSession spark = builder.getOrCreate();
return spark;
}
+
+ public static String getMatrixAsString(double[][] matrix) {
+ try {
+ return DataConverter.toString(DataConverter.convertToMatrixBlock(matrix));
+ } catch (DMLRuntimeException e) {
+ return "N/A";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/3d4c21b0/src/test/java/org/apache/sysml/test/integration/mlcontext/algorithms/MLContextUnivariateStatisticsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/algorithms/MLContextUnivariateStatisticsTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/algorithms/MLContextUnivariateStatisticsTest.java
new file mode 100644
index 0000000..c18c20f
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/algorithms/MLContextUnivariateStatisticsTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.mlcontext.algorithms;
+
+import static org.apache.sysml.api.mlcontext.ScriptFactory.dmlFromFile;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.log4j.Logger;
+import org.apache.sysml.api.mlcontext.Script;
+import org.apache.sysml.test.integration.mlcontext.MLContextTestBase;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MLContextUnivariateStatisticsTest extends MLContextTestBase {
+ protected static Logger log = Logger.getLogger(MLContextUnivariateStatisticsTest.class);
+
+ protected final static String TEST_SCRIPT = "scripts/algorithms/Univar-Stats.dml";
+
+ @Test
+ public void testRandomMatrix() {
+ double[][] random10x3 = getRandomMatrix(10, 3, 0.0, 10.0, 0.9, -1);
+ double[][] types = new double[][] { { 1.0, 1.0, 1.0 } };
+ Script univarStats = dmlFromFile(TEST_SCRIPT);
+ univarStats.in("A", random10x3).in("K", types).in("$CONSOLE_OUTPUT", true).out("baseStats");
+ ml.execute(univarStats);
+ }
+
+ @Test
+ public void testRandomMatrixWithRandomCategoricalColumn() {
+ double[][] random10x3 = getRandomMatrix(10, 3, 0.0, 10.0, 0.9, -1);
+ log.debug("Matrix before random int column replace:\n" + getMatrixAsString(random10x3));
+ replaceColumnWithRandomInts(random10x3, 2, 1, 2);
+ log.debug("Matrix after random int column replace:\n" + getMatrixAsString(random10x3));
+ double[][] types = new double[][] { { 1.0, 1.0, 2.0 } };
+ Script univarStats = dmlFromFile(TEST_SCRIPT);
+ univarStats.in("A", random10x3).in("K", types).out("baseStats");
+ ml.execute(univarStats);
+ }
+
+ @Test
+ public void testScaleColumn() {
+ double[][] matrix = new double[][] { { 1.0 }, { 2.0 }, { 2.0 }, { 3.0 }, { 4.0 } };
+ double[][] types = new double[][] { { 1.0 } };
+ Script univarStats = dmlFromFile(TEST_SCRIPT);
+ univarStats.in("A", matrix).in("K", types).out("baseStats");
+ double[][] stats = ml.execute(univarStats).getMatrix("baseStats").to2DDoubleArray();
+ log.debug("Stats for scale column:\n" + getMatrixAsString(stats));
+ Assert.assertEquals(1.0, stats[0][0], 0); // minimum
+ Assert.assertEquals(4.0, stats[1][0], 0); // maximum
+ Assert.assertEquals(2.4, stats[3][0], 0); // average
+ Assert.assertEquals(2.0, stats[12][0], 0); // mean
+ }
+
+ @Test
+ public void testCategoricalColumn() {
+ double[][] matrix = new double[][] { { 1.0 }, { 2.0 }, { 2.0 }, { 3.0 }, { 4.0 } };
+ double[][] types = new double[][] { { 2.0 } };
+ Script univarStats = dmlFromFile(TEST_SCRIPT);
+ univarStats.in("A", matrix).in("K", types).out("baseStats");
+ double[][] stats = ml.execute(univarStats).getMatrix("baseStats").to2DDoubleArray();
+ log.debug("Stats for categorical column:\n" + getMatrixAsString(stats));
+ Assert.assertEquals(4.0, stats[14][0], 0); // number of categories
+ Assert.assertEquals(2.0, stats[15][0], 0); // mode
+ Assert.assertEquals(1.0, stats[16][0], 0); // number of modes
+ }
+
+ private void replaceColumnWithRandomInts(double[][] matrix, int whichColumn, int lowValue, int highValue) {
+ for (int i = 0; i < matrix.length; i++) {
+ double[] row = matrix[i];
+ row[whichColumn] = ThreadLocalRandom.current().nextInt(lowValue, highValue + 1);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/3d4c21b0/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/algorithms/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/algorithms/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/algorithms/ZPackageSuite.java
new file mode 100644
index 0000000..2230625
--- /dev/null
+++ b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/algorithms/ZPackageSuite.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.mlcontext.algorithms;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Group together the tests in this package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({ MLContextUnivariateStatisticsTest.class })
+
+/** This class is just a holder for the above JUnit annotations. */
+public class ZPackageSuite {
+
+}