You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/04/23 07:26:54 UTC
[1/2] incubator-systemml git commit: [MINOR] Remove unused tests and
test utility functions
Repository: incubator-systemml
Updated Branches:
refs/heads/master 56e6b6e94 -> c8007f454
[MINOR] Remove unused tests and test utility functions
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/edd3491f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/edd3491f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/edd3491f
Branch: refs/heads/master
Commit: edd3491fa07e8897b2f119f438b88e8bf769f100
Parents: 56e6b6e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Apr 22 22:29:40 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Apr 22 22:29:40 2017 -0700
----------------------------------------------------------------------
.../AutomatedScalabilityTestBase.java | 93 ---------
.../test/integration/AutomatedTestBase.java | 14 --
.../BinaryMatrixCharacteristics.java | 86 ---------
.../integration/TestMatrixCharacteristics.java | 102 ----------
.../integration/functions/data/WriteTest.java | 28 +--
.../scalability/LinearRegressionTest.java | 57 ------
.../integration/scalability/PageRankTest.java | 56 ------
.../org/apache/sysml/test/utils/TestUtils.java | 187 +------------------
.../integration/scalability/ZPackageSuite.java | 37 ----
9 files changed, 23 insertions(+), 637 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
deleted file mode 100644
index 0624c55..0000000
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration;
-
-import java.util.ArrayList;
-
-import org.junit.After;
-import org.junit.Before;
-
-import org.apache.sysml.test.utils.TestUtils;
-
-
-public abstract class AutomatedScalabilityTestBase extends AutomatedTestBase
-{
-
-
- private static final boolean RUN_SCALABILITY_TESTS = false;
-
- private long[] timeMeasurements;
- protected int[][] matrixSizes;
- protected ArrayList<TestMatrixCharacteristics> inputMatrices;
-
-
- @Before
- public void setUpScalabilityTest() {
- inputMatrices = new ArrayList<TestMatrixCharacteristics>();
- }
-
- public abstract void setUp();
-
- protected void runTest() {
- if(!RUN_SCALABILITY_TESTS)
- return;
-
- timeMeasurements = new long[matrixSizes.length];
- for(int i = 0; i < matrixSizes.length; i++) {
- for(TestMatrixCharacteristics inputMatrix : inputMatrices) {
- if(inputMatrix.getRows() == -1)
- inputMatrix.setRows(matrixSizes[i][inputMatrix.getRowsIndexInMatrixSizes()]);
- if(inputMatrix.getCols() == -1)
- inputMatrix.setCols(matrixSizes[i][inputMatrix.getColsIndexInMatrixSizes()]);
- createRandomMatrix(inputMatrix);
- }
-
- for(int j = 0; j < matrixSizes[i].length; j++) {
- testVariables.put(Integer.toString(j), Integer.toString(matrixSizes[i][j]));
- }
-
- long startingTime = System.currentTimeMillis();
- super.runTest();
- long finishingTime = System.currentTimeMillis();
- timeMeasurements[i] = (finishingTime - startingTime);
-
- TestUtils.renameTempDMLScript(baseDirectory + selectedTest + ".dml");
- }
- }
-
- protected TestMatrixCharacteristics addInputMatrix(String name, int rows, int cols, double min, double max,
- double sparsity, long seed) {
- TestMatrixCharacteristics inputMatrix = new TestMatrixCharacteristics(name, rows, cols, min, max,
- sparsity, seed);
- inputMatrices.add(inputMatrix);
- return inputMatrix;
- }
-
- @After
- public void displayTimeMeasurements() {
- if(!RUN_SCALABILITY_TESTS)
- return;
-
- for(long timeMeasurement : timeMeasurements) {
- System.out.println("measured time: " + timeMeasurement);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index d1cba1d..a822ae7 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -479,20 +479,6 @@ public abstract class AutomatedTestBase
return matrix;
}
- /**
- * <p>
- * Generates a random matrix with the specified characteristics and writes
- * it to a file.
- * </p>
- *
- * @param matrix
- * matrix characteristics
- */
- protected void createRandomMatrix(TestMatrixCharacteristics matrix) {
- createRandomMatrix(matrix.getMatrixName(), matrix.getRows(), matrix.getCols(), matrix.getMinValue(), matrix
- .getMaxValue(), matrix.getSparsity(), matrix.getSeed());
- }
-
private void cleanupExistingData(String fname, boolean cleanupRData) throws IOException {
MapReduceTool.deleteFileIfExistOnHDFS(fname);
MapReduceTool.deleteFileIfExistOnHDFS(fname + ".mtd");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
deleted file mode 100644
index c36c737..0000000
--- a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration;
-
-
-/**
- * <p>Contains characteristics about a binary matrix.</p>
- *
- *
- */
-public class BinaryMatrixCharacteristics
-{
-
-
- private double[][] values;
- private int rows;
- private int cols;
- private int rowsInBlock;
- private int rowsInLastBlock;
- private int colsInBlock;
- private int colsInLastBlock;
- private long nonZeros;
-
-
- public BinaryMatrixCharacteristics(double[][] values, int rows, int cols, int rowsInBlock, int rowsInLastBlock,
- int colsInBlock, int colsInLastBlock, long nonZeros) {
- this.values = values;
- this.rows = rows;
- this.cols = cols;
- this.rowsInBlock = rowsInBlock;
- this.rowsInLastBlock = rowsInLastBlock;
- this.colsInBlock = colsInBlock;
- this.colsInLastBlock = colsInLastBlock;
- this.nonZeros = nonZeros;
- }
-
- public double[][] getValues() {
- return values;
- }
-
- public int getRows() {
- return rows;
- }
-
- public int getCols() {
- return cols;
- }
-
- public int getRowsInBlock() {
- return rowsInBlock;
- }
-
- public int getRowsInLastBlock() {
- return rowsInLastBlock;
- }
-
- public int getColsInBlock() {
- return colsInBlock;
- }
-
- public int getColsInLastBlock() {
- return colsInLastBlock;
- }
-
- public long getNonZeros() {
- return nonZeros;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
deleted file mode 100644
index 38c51a7..0000000
--- a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration;
-
-public class TestMatrixCharacteristics
-{
-
-
- private String matrixName;
- private int rows;
- private int rowsIndexInMatrixSizes;
- private int cols;
- private int colsIndexInMatrixSizes;
- private double minValue;
- private double maxValue;
- private double sparsity;
- private long seed;
-
-
- public TestMatrixCharacteristics(String matrixName, int rows, int cols, double minValue, double maxValue,
- double sparsity, long seed) {
- this.matrixName = matrixName;
- this.rows = rows;
- this.cols = cols;
- this.minValue = minValue;
- this.maxValue = maxValue;
- this.sparsity = sparsity;
- this.seed = seed;
- }
-
- public String getMatrixName() {
- return matrixName;
- }
-
- public int getRows() {
- return rows;
- }
-
- public void setRows(int rows) {
- this.rows = rows;
- }
-
- public int getRowsIndexInMatrixSizes() {
- return rowsIndexInMatrixSizes;
- }
-
- public TestMatrixCharacteristics setRowsIndexInMatrixSizes(int rowsIndexInMatrixSizes) {
- this.rowsIndexInMatrixSizes = rowsIndexInMatrixSizes;
- return this;
- }
-
- public int getCols() {
- return cols;
- }
-
- public void setCols(int cols) {
- this.cols = cols;
- }
-
- public int getColsIndexInMatrixSizes() {
- return colsIndexInMatrixSizes;
- }
-
- public TestMatrixCharacteristics setColsIndexInMatrixSizes(int colsIndexInMatrixSizes) {
- this.colsIndexInMatrixSizes = colsIndexInMatrixSizes;
- return this;
- }
-
- public double getMinValue() {
- return minValue;
- }
-
- public double getMaxValue() {
- return maxValue;
- }
-
- public double getSparsity() {
- return sparsity;
- }
-
- public long getSeed() {
- return seed;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
index 448893c..441c9c7 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java
@@ -19,12 +19,13 @@
package org.apache.sysml.test.integration.functions.data;
-import static org.junit.Assert.*;
-
import org.junit.Test;
-
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.test.integration.BinaryMatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
@@ -96,16 +97,15 @@ public class WriteTest extends AutomatedTestBase
runTest();
- //compareResults();
-
- BinaryMatrixCharacteristics matrix = TestUtils.readBlocksFromSequenceFile(output("a"),1000,1000);
- assertEquals(rows, matrix.getRows());
- assertEquals(cols, matrix.getCols());
- double[][] matrixValues = matrix.getValues();
- for(int i = 0; i < rows; i++) {
- for(int j = 0; j < cols; j++) {
- assertEquals(i + "," + j, a[i][j], matrixValues[i][j], 0);
- }
+ //read and compare output matrix
+ try {
+ MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo);
+ MatrixBlock mb = reader.readMatrixFromHDFS(output("a"), rows, cols, 1000, 1000, -1);
+ checkDMLMetaDataFile("a", new MatrixCharacteristics(rows,cols,1000,1000));
+ TestUtils.compareMatrices(a, DataConverter.convertToDoubleMatrix(mb), rows, cols, 0);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
deleted file mode 100644
index 9795245..0000000
--- a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration.scalability;
-
-import org.junit.Test;
-
-import org.apache.sysml.test.integration.AutomatedScalabilityTestBase;
-import org.apache.sysml.test.integration.TestConfiguration;
-import org.apache.sysml.test.utils.TestUtils;
-
-
-
-public class LinearRegressionTest extends AutomatedScalabilityTestBase
-{
-
- private static final String TEST_DIR = "test/scripts/scalability/linear_regression/";
- private static final String TEST_CLASS_DIR = TEST_DIR + LinearRegressionTest.class.getSimpleName() + "/";
-
-
- @Override
- public void setUp() {
- TestUtils.clearAssertionInformation();
- addTestConfiguration("LinearRegressionTest", new TestConfiguration(TEST_CLASS_DIR, "LinearRegressionTest", new String[] { "w" }));
- matrixSizes = new int[][] {
- { 19004, 15436 }
- };
- }
-
- @Test
- public void testLinearRegression() {
- TestConfiguration config = getTestConfiguration("LinearRegressionTest");
- loadTestConfiguration(config);
-
- addInputMatrix("g", -1, -1, 0, 1, 0.00594116, -1).setRowsIndexInMatrixSizes(0).setColsIndexInMatrixSizes(1);
- addInputMatrix("b", -1, 1, 1, 10, 1, -1).setRowsIndexInMatrixSizes(0);
-
- runTest();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
deleted file mode 100644
index d8159ee..0000000
--- a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration.scalability;
-
-import org.junit.Test;
-
-import org.apache.sysml.test.integration.AutomatedScalabilityTestBase;
-import org.apache.sysml.test.integration.TestConfiguration;
-
-
-
-public class PageRankTest extends AutomatedScalabilityTestBase
-{
- private static final String TEST_DIR = "test/scripts/scalability/page_rank/";
- private static final String TEST_CLASS_DIR = TEST_DIR + PageRankTest.class.getSimpleName() + "/";
-
-
- @Override
- public void setUp() {
- addTestConfiguration("PageRankTest", new TestConfiguration(TEST_CLASS_DIR, "PageRankTest", new String[] { "p" }));
- matrixSizes = new int[][] {
- { 9914 }
- };
- }
-
- @Test
- public void testPageRank() {
- TestConfiguration config = getTestConfiguration("PageRankTest");
- loadTestConfiguration(config);
-
- addInputMatrix("g", -1, -1, 1, 1, 0.000374962, -1).setRowsIndexInMatrixSizes(0).setColsIndexInMatrixSizes(0);
- addInputMatrix("p", -1, 1, 1, 1, 1, -1).setRowsIndexInMatrixSizes(0);
- addInputMatrix("e", -1, 1, 1, 1, 1, -1).setRowsIndexInMatrixSizes(0);
- addInputMatrix("u", 1, -1, 1, 1, 1, -1).setColsIndexInMatrixSizes(0);
-
- runTest();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index 2d1a4c1..19b741d 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -40,7 +40,6 @@ import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
@@ -58,8 +57,8 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.io.FrameWriter;
import org.apache.sysml.runtime.io.FrameWriterFactory;
import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
-import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
@@ -67,7 +66,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.test.integration.AutomatedTestBase;
-import org.apache.sysml.test.integration.BinaryMatrixCharacteristics;
/**
@@ -1048,8 +1046,10 @@ public class TestUtils
* Checks a matrix against a number of specifications.
* </p>
*
- * @param matrix
- * matrix
+ * @param data
+ * matrix data
+ * @param mc
+ * matrix characteristics
* @param rows
* number of rows
* @param cols
@@ -1059,14 +1059,13 @@ public class TestUtils
* @param max
* maximum value
*/
- public static void checkMatrix(BinaryMatrixCharacteristics matrix, long rows, long cols, double min, double max) {
- assertEquals(rows, matrix.getRows());
- assertEquals(cols, matrix.getCols());
- double[][] matrixValues = matrix.getValues();
+ public static void checkMatrix(double[][] data, MatrixCharacteristics mc, long rows, long cols, double min, double max) {
+ assertEquals(rows, mc.getRows());
+ assertEquals(cols, mc.getCols());
for (int i = 0; i < rows; i++) {
for (int j = 0; j < cols; j++) {
assertTrue("invalid value",
- ((matrixValues[i][j] >= min && matrixValues[i][j] <= max) || matrixValues[i][j] == 0));
+ ((data[i][j] >= min && data[i][j] <= max) || data[i][j] == 0));
}
}
}
@@ -1819,174 +1818,6 @@ public class TestUtils
/**
* <p>
- * Reads binary cells from a file. A matrix characteristic is created which
- * contains the characteristics of the matrix read from the file and the
- * values.
- * </p>
- *
- * @param directory
- * directory containing the matrix
- * @return matrix characteristics
- */
- @SuppressWarnings("deprecation")
- public static BinaryMatrixCharacteristics readCellsFromSequenceFile(String directory) {
- try {
- FileSystem fs = FileSystem.get(conf);
- FileStatus[] files = fs.listStatus(new Path(directory));
-
- HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>();
- int rows = 0;
- int cols = 0;
- MatrixIndexes indexes = new MatrixIndexes();
- MatrixCell value = new MatrixCell();
- for (FileStatus file : files) {
- SequenceFile.Reader reader = null;
- try {
- reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf);
- while (reader.next(indexes, value)) {
- if (rows < indexes.getRowIndex())
- rows = (int) indexes.getRowIndex();
- if (cols < indexes.getColumnIndex())
- cols = (int) indexes.getColumnIndex();
- valueMap.put(new MatrixIndexes(indexes), value.getValue());
- }
- }
- finally {
- IOUtilFunctions.closeSilently(reader);
- }
- }
-
- double[][] values = new double[rows][cols];
- long nonZeros = 0;
- for (MatrixIndexes index : valueMap.keySet()) {
- values[(int)index.getRowIndex() - 1][(int)index.getColumnIndex() - 1] = valueMap.get(index);
- if (valueMap.get(index) != 0)
- nonZeros++;
- }
-
- return new BinaryMatrixCharacteristics(values, rows, cols, 0, 0, 0, 0, nonZeros);
- } catch (IOException e) {
- e.printStackTrace();
- fail("unable to read sequence file in " + directory);
- }
-
- return null;
- }
-
- /**
- * <p>
- * Reads binary blocks from a file. A matrix characteristic is created which
- * contains the characteristics of the matrix read from the file and the
- * values.
- * </p>
- *
- * @param directory
- * directory containing the matrix
- * @param rowsInBlock
- * rows in block
- * @param colsInBlock
- * columns in block
- * @return matrix characteristics
- */
- @SuppressWarnings("deprecation")
- public static BinaryMatrixCharacteristics readBlocksFromSequenceFile(String directory, int rowsInBlock,
- int colsInBlock) {
- try {
- FileSystem fs = FileSystem.get(conf);
- FileStatus[] files = fs.listStatus(new Path(directory));
-
- HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>();
- int rowsInLastBlock = -1;
- int colsInLastBlock = -1;
- int rows = 0;
- int cols = 0;
- MatrixIndexes indexes = new MatrixIndexes();
- MatrixBlock value = new MatrixBlock();
- for (FileStatus file : files) {
- SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf);
-
- try {
- while (reader.next(indexes, value)) {
- if (value.getNumRows() < rowsInBlock) {
- if (rowsInLastBlock == -1)
- rowsInLastBlock = value.getNumRows();
- else if (rowsInLastBlock != value.getNumRows())
- fail("invalid block sizes");
- rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows());
- } else if (value.getNumRows() == rowsInBlock) {
- if (rows <= (indexes.getRowIndex() * rowsInBlock + value.getNumRows())) {
- if (rowsInLastBlock == -1)
- rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows());
- else
- fail("invalid block sizes");
- }
- } else {
- fail("invalid block sizes");
- }
-
- if (value.getNumColumns() < colsInBlock) {
- if (colsInLastBlock == -1)
- colsInLastBlock = value.getNumColumns();
- else if (colsInLastBlock != value.getNumColumns())
- fail("invalid block sizes");
- cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns());
- } else if (value.getNumColumns() == colsInBlock) {
- if (cols <= (indexes.getColumnIndex() * colsInBlock + value.getNumColumns())) {
- if (colsInLastBlock == -1)
- cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns());
- else
- fail("invalid block sizes");
- }
- } else {
- fail("invalid block sizes");
- }
-
- if (value.isInSparseFormat()) {
- Iterator<IJV> iter = value.getSparseBlockIterator();
- while( iter.hasNext() )
- {
- IJV cell = iter.next();
- valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + cell.getI()),
- (int) ((indexes.getColumnIndex() - 1) * colsInBlock + cell.getJ())), cell.getV());
- }
-
- } else {
- double[] valuesInBlock = value.getDenseBlock();
- for (int i = 0; i < value.getNumRows(); i++) {
- for (int j = 0; j < value.getNumColumns(); j++) {
- valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i),
- (int) ((indexes.getColumnIndex() - 1) * colsInBlock + j)), valuesInBlock[i
- * value.getNumColumns() + j]);
- }
- }
- }
- }
- }
- finally {
- IOUtilFunctions.closeSilently(reader);
- }
- }
-
- long nonZeros = 0;
- double[][] values = new double[rows][cols];
- for (MatrixIndexes index : valueMap.keySet()) {
- values[(int)index.getRowIndex()][(int)index.getColumnIndex()] = valueMap.get(index);
- if (valueMap.get(index) != 0)
- nonZeros++;
- }
-
- return new BinaryMatrixCharacteristics(values, rows, cols, rowsInBlock, rowsInLastBlock, colsInBlock,
- colsInLastBlock, nonZeros);
- } catch (IOException e) {
- e.printStackTrace();
- fail("unable to read sequence file in " + directory);
- }
-
- return null;
- }
-
- /**
- * <p>
* Returns the path to a file in a directory if it is the only file in the
* directory.
* </p>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
deleted file mode 100644
index f0e3ae8..0000000
--- a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration.scalability;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/** Group together the tests in this package into a single suite so that the Maven build
- * won't run two of them at once. */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- LinearRegressionTest.class,
- PageRankTest.class
-})
-
-
-/** This class is just a holder for the above JUnit annotations. */
-public class ZPackageSuite {
-
-}
[2/2] incubator-systemml git commit: [SYSTEMML-1555] Decouple literal
replacement (scalar-only, in-place)
Posted by mb...@apache.org.
[SYSTEMML-1555] Decouple literal replacement (scalar-only, in-place)
So far full literal replacement was only applied for recompilations that
are not in-place, i.e., compilations that work over a deep copy of the
hop dag. Furthermore, we always applied full literal replacement
including the replacement of small matrix operations with its scalar
results.
This patch decouples these aspects in order to allow fine-grained
external control over literal replacement.
Furthermore, this patch also fixes the codegen MSVM and L2SVM tests to
account for the recently changed CLI command line argument parsing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c8007f45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c8007f45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c8007f45
Branch: refs/heads/master
Commit: c8007f4548c0f0f0a5f7ba4f8c785871b6d31fe8
Parents: edd3491
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Apr 23 00:16:46 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Apr 23 00:16:46 2017 -0700
----------------------------------------------------------------------
.../sysml/hops/codegen/SpoofCompiler.java | 4 +-
.../hops/recompile/LiteralReplacement.java | 22 ++++----
.../apache/sysml/hops/recompile/Recompiler.java | 54 ++++++++++++--------
.../runtime/controlprogram/ProgramBlock.java | 6 ++-
.../parfor/opt/OptimizerRuleBased.java | 3 +-
.../parfor/opt/ProgramRecompiler.java | 10 ++--
.../sysml/yarn/ropt/ResourceOptimizer.java | 38 +++++++++-----
.../functions/codegen/AlgorithmL2SVM.java | 2 +-
.../functions/codegen/AlgorithmMSVM.java | 2 +-
9 files changed, 85 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
index 816b7ae..aa5e7e3 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java
@@ -293,7 +293,7 @@ public class SpoofCompiler
{
//create copy of hop dag, call codegen, and generate instructions
return Recompiler.recompileHopsDag(sb, roots,
- new LocalVariableMap(), new RecompileStatus(), false, 0);
+ new LocalVariableMap(), new RecompileStatus(), false, false, 0);
}
public static ArrayList<Instruction> generateCodeFromHopDAGsToInst(Hop root)
@@ -301,7 +301,7 @@ public class SpoofCompiler
{
//create copy of hop dag, call codegen, and generate instructions
return Recompiler.recompileHopsDag(root,
- new LocalVariableMap(), new RecompileStatus(), false, 0);
+ new LocalVariableMap(), new RecompileStatus(), false, false, 0);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java b/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
index f539abc..d7b97e9 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/LiteralReplacement.java
@@ -50,7 +50,7 @@ public class LiteralReplacement
private static final long REPLACE_LITERALS_MAX_MATRIX_SIZE = 1000000; //10^6 cells (8MB)
private static final boolean REPORT_LITERAL_REPLACE_OPS_STATS = true;
- protected static void rReplaceLiterals( Hop hop, LocalVariableMap vars )
+ protected static void rReplaceLiterals( Hop hop, LocalVariableMap vars, boolean scalarsOnly )
throws DMLRuntimeException
{
if( hop.isVisited() )
@@ -68,10 +68,12 @@ public class LiteralReplacement
lit = (lit==null) ? replaceLiteralScalarRead(c, vars) : lit;
lit = (lit==null) ? replaceLiteralValueTypeCastScalarRead(c, vars) : lit;
lit = (lit==null) ? replaceLiteralValueTypeCastLiteral(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralDataTypeCastMatrixRead(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralValueTypeCastRightIndexing(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralFullUnaryAggregate(c, vars) : lit;
- lit = (lit==null) ? replaceLiteralFullUnaryAggregateRightIndexing(c, vars) : lit;
+ if( !scalarsOnly ) {
+ lit = (lit==null) ? replaceLiteralDataTypeCastMatrixRead(c, vars) : lit;
+ lit = (lit==null) ? replaceLiteralValueTypeCastRightIndexing(c, vars) : lit;
+ lit = (lit==null) ? replaceLiteralFullUnaryAggregate(c, vars) : lit;
+ lit = (lit==null) ? replaceLiteralFullUnaryAggregateRightIndexing(c, vars) : lit;
+ }
//replace hop w/ literal on demand
if( lit != null )
@@ -88,15 +90,13 @@ public class LiteralReplacement
}
}
else { //current hop is only parent
- HopRewriteUtils.removeChildReferenceByPos(hop, c, i);
- HopRewriteUtils.addChildReference(hop, lit, i);
+ HopRewriteUtils.replaceChildReference(hop, c, lit, i);
}
}
//recursively process children
- else
- {
- rReplaceLiterals(c, vars);
- }
+ else {
+ rReplaceLiterals(c, vars, scalarsOnly);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 6c99d16..6a2d88c 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -154,6 +154,7 @@ public class Recompiler
* @param vars local variable map
* @param status the recompile status
* @param inplace true if in place
+ * @param litreplace true if literal replacement
* @param tid thread id
* @return list of instructions
* @throws DMLRuntimeException if DMLRuntimeException occurs
@@ -161,7 +162,8 @@ public class Recompiler
* @throws LopsException if LopsException occurs
* @throws IOException if IOException occurs
*/
- public static ArrayList<Instruction> recompileHopsDag( StatementBlock sb, ArrayList<Hop> hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, long tid )
+ public static ArrayList<Instruction> recompileHopsDag( StatementBlock sb, ArrayList<Hop> hops,
+ LocalVariableMap vars, RecompileStatus status, boolean inplace, boolean litreplace, long tid )
throws DMLRuntimeException, HopsException, LopsException, IOException
{
ArrayList<Instruction> newInst = null;
@@ -186,10 +188,10 @@ public class Recompiler
}
// replace scalar reads with literals
- if( !inplace ) {
+ if( !inplace && litreplace ) {
Hop.resetVisitStatus(hops);
for( Hop hopRoot : hops )
- rReplaceLiterals( hopRoot, vars );
+ rReplaceLiterals( hopRoot, vars, false );
}
// refresh matrix characteristics (update stats)
@@ -268,6 +270,7 @@ public class Recompiler
* @param vars local variable map
* @param status recompile status
* @param inplace true if in place
+ * @param litreplace true if literal replacement
* @param tid thread id
* @return list of instructions
* @throws DMLRuntimeException if DMLRuntimeException occurs
@@ -275,7 +278,8 @@ public class Recompiler
* @throws LopsException if LopsException occurs
* @throws IOException if IOException occurs
*/
- public static ArrayList<Instruction> recompileHopsDag( Hop hops, LocalVariableMap vars, RecompileStatus status, boolean inplace, long tid )
+ public static ArrayList<Instruction> recompileHopsDag( Hop hops, LocalVariableMap vars,
+ RecompileStatus status, boolean inplace, boolean litreplace, long tid )
throws DMLRuntimeException, HopsException, LopsException, IOException
{
ArrayList<Instruction> newInst = null;
@@ -299,9 +303,9 @@ public class Recompiler
}
// replace scalar reads with literals
- if( !inplace ) {
+ if( !inplace && litreplace ) {
hops.resetVisitStatus();
- rReplaceLiterals( hops, vars );
+ rReplaceLiterals( hops, vars, false );
}
// refresh matrix characteristics (update stats)
@@ -864,7 +868,8 @@ public class Recompiler
//&& Recompiler.requiresRecompilation( sb.get_hops() )
/*&& !Recompiler.containsNonRecompileInstructions(tmp)*/ )
{
- tmp = Recompiler.recompileHopsDag(sb, sb.get_hops(), vars, status, true, tid);
+ tmp = Recompiler.recompileHopsDag(
+ sb, sb.get_hops(), vars, status, true, false, tid);
pb.setInstructions( tmp );
//propagate stats across hops (should be executed on clone of vars)
@@ -1098,10 +1103,9 @@ public class Recompiler
private static MatrixObject createOutputMatrix( long dim1, long dim2, long nnz )
{
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
+ int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(
- dim1, dim2,
- ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
- nnz);
+ dim1, dim2, blksz, blksz, nnz);
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
moOut.setMetaData(meta);
@@ -1118,7 +1122,8 @@ public class Recompiler
{
Hop hops = isb.getPredicateHops();
if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(hops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ hops, vars, status, true, false, tid);
ipb.setPredicate( tmp );
if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
&& resetRecompile )
@@ -1141,7 +1146,8 @@ public class Recompiler
{
Hop hops = wsb.getPredicateHops();
if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(hops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ hops, vars, status, true, false, tid);
wpb.setPredicate( tmp );
if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
&& resetRecompile )
@@ -1171,17 +1177,20 @@ public class Recompiler
&& resetRecompile )
{
if( fromHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ fromHops, vars, status, true, false, tid);
fpb.setFromInstructions(tmp);
Hop.resetRecompilationFlag(fromHops,ExecType.CP);
}
if( toHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ toHops, vars, status, true, false, tid);
fpb.setToInstructions(tmp);
Hop.resetRecompilationFlag(toHops,ExecType.CP);
}
if( incrHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ incrHops, vars, status, true, false, tid);
fpb.setIncrementInstructions(tmp);
Hop.resetRecompilationFlag(incrHops,ExecType.CP);
}
@@ -1190,15 +1199,18 @@ public class Recompiler
else //no reset of recompilation flags
{
if( fromHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ fromHops, vars, status, true, false, tid);
fpb.setFromInstructions(tmp);
}
if( toHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ toHops, vars, status, true, false, tid);
fpb.setToInstructions(tmp);
}
if( incrHops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, tid);
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ incrHops, vars, status, true, false, tid);
fpb.setIncrementInstructions(tmp);
}
}
@@ -1608,13 +1620,15 @@ public class Recompiler
*
* @param hop high-level operator
* @param vars local variable map
+ * @param scalarsOnly if true, replace only scalar variables but no matrix operations;
+ * if false, apply full literal replacement
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
- public static void rReplaceLiterals( Hop hop, LocalVariableMap vars )
+ public static void rReplaceLiterals( Hop hop, LocalVariableMap vars, boolean scalarsOnly )
throws DMLRuntimeException
{
//public interface
- LiteralReplacement.rReplaceLiterals(hop, vars);
+ LiteralReplacement.rReplaceLiterals(hop, vars, scalarsOnly);
}
public static void rSetExecType( Hop hop, ExecType etype )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index 1847847..71fb1c2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -145,7 +145,8 @@ public class ProgramBlock
&& _sb != null
&& _sb.requiresRecompilation() )
{
- tmp = Recompiler.recompileHopsDag(_sb, _sb.get_hops(), ec.getVariables(), null, false, _tid);
+ tmp = Recompiler.recompileHopsDag(
+ _sb, _sb.get_hops(), ec.getVariables(), null, false, true, _tid);
if( MLContextProxy.isActive() )
tmp = MLContextProxy.performCleanupAfterRecompilation(tmp);
@@ -188,7 +189,8 @@ public class ProgramBlock
if( ConfigurationManager.isDynamicRecompilation()
&& requiresRecompile )
{
- tmp = Recompiler.recompileHopsDag(hops, ec.getVariables(), null, false, _tid);
+ tmp = Recompiler.recompileHopsDag(
+ hops, ec.getVariables(), null, false, true, _tid);
}
if( DMLScript.STATISTICS ){
long t1 = System.nanoTime();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 1a8122a..bbd8fb8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -786,7 +786,8 @@ public class OptimizerRuleBased extends Optimizer
HashMap<Hop, Double> estRix = getPartitionedRIXEstimates(nParent);
//construct new instructions
- ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.get_hops(), vars, null, false, 0);
+ ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(
+ sb, sb.get_hops(), vars, null, false, false, 0);
pb.setInstructions( newInst );
//reset all rix estimated (modified by recompile)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index f6ccc13..2d03af3 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -183,7 +183,8 @@ public class ProgramRecompiler
if( ret )
{
//construct new instructions
- ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.get_hops(), ec.getVariables(), null, true, 0);
+ ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(
+ sb, sb.get_hops(), ec.getVariables(), null, true, false, 0);
pb.setInstructions( newInst );
}
}
@@ -251,7 +252,7 @@ public class ProgramRecompiler
//replace constant literals
Hop.resetVisitStatus(hops);
for( Hop hopRoot : hops )
- Recompiler.rReplaceLiterals( hopRoot, vars );
+ Recompiler.rReplaceLiterals( hopRoot, vars, true );
}
}
}
@@ -261,7 +262,7 @@ public class ProgramRecompiler
{
if( pred != null ){
pred.resetVisitStatus();
- Recompiler.rReplaceLiterals(pred, vars);
+ Recompiler.rReplaceLiterals(pred, vars, true);
}
}
@@ -380,7 +381,8 @@ public class ProgramRecompiler
if( ret )
{
//construct new instructions
- tmp = Recompiler.recompileHopsDag(hop, ec.getVariables(), null, true, 0);
+ tmp = Recompiler.recompileHopsDag(
+ hop, ec.getVariables(), null, true, false, 0);
}
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
index d753406..8b6003f 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
@@ -207,7 +207,8 @@ public class ResourceOptimizer
WhileProgramBlock wpb = (WhileProgramBlock)pb;
WhileStatementBlock sb = (WhileStatementBlock) pb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null && sb.getPredicateHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
wpb.setPredicate( inst );
B.add(wpb);
_cntCompilePB ++;
@@ -219,7 +220,8 @@ public class ResourceOptimizer
IfProgramBlock ipb = (IfProgramBlock)pb;
IfStatementBlock sb = (IfStatementBlock) ipb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null && sb.getPredicateHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
ipb.setPredicate( inst );
B.add(ipb);
_cntCompilePB ++;
@@ -233,15 +235,18 @@ public class ResourceOptimizer
ForStatementBlock sb = (ForStatementBlock) fpb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null ){
if( sb.getFromHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getFromHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getFromHops(), new LocalVariableMap(), null, false, false, 0);
fpb.setFromInstructions( inst );
}
if( sb.getToHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getToHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getToHops(), new LocalVariableMap(), null, false, false, 0);
fpb.setToInstructions( inst );
}
if( sb.getIncrementHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getIncrementHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getIncrementHops(), new LocalVariableMap(), null, false, false, 0);
fpb.setIncrementInstructions( inst );
}
B.add(fpb);
@@ -252,8 +257,8 @@ public class ResourceOptimizer
else
{
StatementBlock sb = pb.getStatementBlock();
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb, sb.get_hops(),
- new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb, sb.get_hops(), new LocalVariableMap(), null, false, false, 0);
pb.setInstructions( inst );
B.add(pb);
_cntCompilePB ++;
@@ -288,7 +293,8 @@ public class ResourceOptimizer
WhileProgramBlock wpb = (WhileProgramBlock)pb;
WhileStatementBlock sb = (WhileStatementBlock) pb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null && sb.getPredicateHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
wpb.setPredicate( inst );
}
@@ -298,7 +304,8 @@ public class ResourceOptimizer
IfProgramBlock ipb = (IfProgramBlock)pb;
IfStatementBlock sb = (IfStatementBlock) ipb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null && sb.getPredicateHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
ipb.setPredicate( inst );
}
@@ -309,17 +316,20 @@ public class ResourceOptimizer
ForStatementBlock sb = (ForStatementBlock) fpb.getStatementBlock();
if( INCLUDE_PREDICATES && sb!=null ){
if( sb.getFromHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getFromHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getFromHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setFromInstructions( inst );
}
if( sb.getToHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getToHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getToHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setToInstructions( inst );
}
if( sb.getIncrementHops()!=null ){
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getIncrementHops(), new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb.getIncrementHops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
fpb.setIncrementInstructions( inst );
}
@@ -328,8 +338,8 @@ public class ResourceOptimizer
else //last-level program blocks
{
StatementBlock sb = pb.getStatementBlock();
- ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb, sb.get_hops(),
- new LocalVariableMap(), null, false, 0);
+ ArrayList<Instruction> inst = Recompiler.recompileHopsDag(
+ sb, sb.get_hops(), new LocalVariableMap(), null, false, false, 0);
inst = annotateMRJobInstructions(inst, cp, mr);
pb.setInstructions( inst );
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmL2SVM.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmL2SVM.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmL2SVM.java
index 6f005ad..2e03ce6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmL2SVM.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmL2SVM.java
@@ -123,7 +123,7 @@ public class AlgorithmL2SVM extends AutomatedTestBase
fullDMLScriptName = "scripts/algorithms/l2-svm.dml";
programArgs = new String[]{ "-explain", "-stats", "-nvargs", "X="+input("X"), "Y="+input("Y"),
"icpt="+String.valueOf(intercept), "tol="+String.valueOf(epsilon), "reg=0.001",
- "maxiter="+String.valueOf(maxiter), "model="+output("w"), "Log=\" \""};
+ "maxiter="+String.valueOf(maxiter), "model="+output("w"), "Log= "};
rCmd = getRCmd(inputDir(), String.valueOf(intercept),String.valueOf(epsilon),
String.valueOf(maxiter), expectedDir());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8007f45/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmMSVM.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmMSVM.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmMSVM.java
index 7f26d70..2a5de13 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmMSVM.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/AlgorithmMSVM.java
@@ -122,7 +122,7 @@ public class AlgorithmMSVM extends AutomatedTestBase
fullDMLScriptName = "scripts/algorithms/m-svm.dml";
programArgs = new String[]{ "-explain", "-stats", "-nvargs", "X="+input("X"), "Y="+input("Y"),
"icpt="+String.valueOf(intercept), "tol="+String.valueOf(epsilon), "reg=0.001",
- "maxiter="+String.valueOf(maxiter), "model="+output("w"), "Log=\" \""};
+ "maxiter="+String.valueOf(maxiter), "model="+output("w"), "Log= "};
rCmd = getRCmd(inputDir(), String.valueOf(intercept),String.valueOf(epsilon),
String.valueOf(maxiter), expectedDir());