You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/05/10 03:08:55 UTC
incubator-systemml git commit: [SYSTEMML-1595] Fix missing block
sizes for mlcontext/jmlc outputs
Repository: incubator-systemml
Updated Branches:
refs/heads/master 5489c665d -> 282d8ecc6
[SYSTEMML-1595] Fix missing block sizes for mlcontext/jmlc outputs
This patch fixes the rewrite for exchanging persistent with transient
writes, which is used in the mlcontext and jmlc APIs. Since the original
persistent write might be an external format without blocksizes, we now
always set the blocksize of the new transient write (which always
requires block sizes) according to write input. Note that this did not
show up before, because the twrite usually just translates to a renaming
of variable names, which preserves the meta data of the passed
intermediate.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/282d8ecc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/282d8ecc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/282d8ecc
Branch: refs/heads/master
Commit: 282d8ecc68befca42c603ff326465c65d92dde13
Parents: 5489c66
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue May 9 20:11:15 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue May 9 20:11:24 2017 -0700
----------------------------------------------------------------------
.../RewriteRemovePersistentReadWrite.java | 2 +
.../mlcontext/MLContextOutputBlocksizeTest.java | 150 +++++++++++++++++++
.../integration/mlcontext/ZPackageSuite.java | 1 +
3 files changed, 153 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/282d8ecc/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
index bd734c9..011eecd 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
@@ -150,6 +150,8 @@ public class RewriteRemovePersistentReadWrite extends HopRewriteRule
case PERSISTENTWRITE:
if( _outputs.contains(dop.getName()) ) {
dop.setDataOpType(DataOpTypes.TRANSIENTWRITE);
+ dop.setRowsInBlock(dop.getInput().get(0).getRowsInBlock());
+ dop.setColsInBlock(dop.getInput().get(0).getColsInBlock());
if (hop.getDataType() == DataType.SCALAR) {
dop.removeInput("iofilename");
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/282d8ecc/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOutputBlocksizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOutputBlocksizeTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOutputBlocksizeTest.java
new file mode 100644
index 0000000..c521173
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOutputBlocksizeTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.mlcontext;
+
+import static org.apache.sysml.api.mlcontext.ScriptFactory.dml;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.sysml.api.mlcontext.BinaryBlockMatrix;
+import org.apache.sysml.api.mlcontext.MLContext;
+import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel;
+import org.apache.sysml.api.mlcontext.MLResults;
+import org.apache.sysml.api.mlcontext.Script;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class MLContextOutputBlocksizeTest extends AutomatedTestBase
+{
+ protected final static String TEST_DIR = "org/apache/sysml/api/mlcontext";
+ protected final static String TEST_NAME = "MLContext";
+
+ private final static int rows = 100;
+ private final static int cols = 63;
+ private final static double sparsity = 0.7;
+
+ private static SparkConf conf;
+ private static JavaSparkContext sc;
+ private static MLContext ml;
+
+ @BeforeClass
+ public static void setUpClass() {
+ if (conf == null)
+ conf = SparkExecutionContext.createSystemMLSparkConf()
+ .setAppName("MLContextTest").setMaster("local");
+ if (sc == null)
+ sc = new JavaSparkContext(conf);
+ ml = new MLContext(sc);
+ }
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_DIR, TEST_NAME);
+ getAndLoadTestConfiguration(TEST_NAME);
+ }
+
+
+ @Test
+ public void testOutputBlocksizeTextcell() {
+ runMLContextOutputBlocksizeTest("text");
+ }
+
+ @Test
+ public void testOutputBlocksizeCSV() {
+ runMLContextOutputBlocksizeTest("csv");
+ }
+
+ @Test
+ public void testOutputBlocksizeMM() {
+ runMLContextOutputBlocksizeTest("mm");
+ }
+
+ @Test
+ public void testOutputBlocksizeBinary() {
+ runMLContextOutputBlocksizeTest("binary");
+ }
+
+
+ private void runMLContextOutputBlocksizeTest(String format)
+ {
+ try
+ {
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 76543);
+ MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
+ int blksz = ConfigurationManager.getBlocksize();
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, blksz, blksz, mbA.getNonZeros());
+
+ //create input dataset
+ JavaPairRDD<MatrixIndexes,MatrixBlock> in = SparkExecutionContext.toMatrixJavaPairRDD(sc, mbA, blksz, blksz);
+ BinaryBlockMatrix bbmatrix = new BinaryBlockMatrix(in, mc);
+
+ ml.setExplain(true);
+ ml.setExplainLevel(ExplainLevel.HOPS);
+
+ //execute script
+ String s ="if( sum(X) > 0 )"
+ + " X = X/2;"
+ + "R = X;"
+ + "write(R, \"/tmp\", format=\""+format+"\");";
+ Script script = dml(s).in("X", bbmatrix).out("R");
+ MLResults results = ml.execute(script);
+
+ //compare output matrix characteristics
+ MatrixCharacteristics mcOut = results.getMatrix("R")
+ .getMatrixMetadata().asMatrixCharacteristics();
+ Assert.assertEquals(blksz, mcOut.getRowsPerBlock());
+ Assert.assertEquals(blksz, mcOut.getColsPerBlock());
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ // stop spark context to allow single jvm tests (otherwise the
+ // next test that tries to create a SparkContext would fail)
+ sc.stop();
+ sc = null;
+ conf = null;
+
+ // clear status mlcontext and spark exec context
+ ml.close();
+ ml = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/282d8ecc/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
index 44f3533..d5a6a21 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
@@ -29,6 +29,7 @@ import org.junit.runners.Suite;
@Suite.SuiteClasses({
MLContextFrameTest.class,
MLContextMultipleScriptsTest.class,
+ MLContextOutputBlocksizeTest.class,
MLContextParforDatasetTest.class,
MLContextScratchCleanupTest.class,
MLContextTest.class