You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/12/16 23:08:17 UTC
incubator-systemml git commit: [SYSTEMML-1156] Fix spark checkpoint
injection before loops
Repository: incubator-systemml
Updated Branches:
refs/heads/master 1ab2d7e68 -> 154f0778c
[SYSTEMML-1156] Fix spark checkpoint injection before loops
This patch fixes the 'spark checkpoint injection rewrite before loops',
which did not handle index identifiers correctly leading to wrong
statically propagated sizes and thus incorrectly applied rewrites such
as the removal of unnecessary right indexing operations. Note that
dynamic recompilation has hidden this issue in various other scripts as
sizes were corrected during runtime.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/154f0778
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/154f0778
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/154f0778
Branch: refs/heads/master
Commit: 154f0778c311728002330499552786e22c1b4286
Parents: 1ab2d7e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Dec 16 23:47:57 2016 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Dec 16 23:48:40 2016 +0100
----------------------------------------------------------------------
.../hops/rewrite/RewriteInjectSparkLoopCheckpointing.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/154f0778/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
index 55a6f15..6b46ee2 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java
@@ -28,6 +28,7 @@ import org.apache.sysml.hops.Hop.DataOpTypes;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.IndexedIdentifier;
import org.apache.sysml.parser.StatementBlock;
import org.apache.sysml.parser.VariableSet;
import org.apache.sysml.parser.WhileStatementBlock;
@@ -96,11 +97,13 @@ public class RewriteInjectSparkLoopCheckpointing extends StatementBlockRewriteRu
for( String var : candidates )
{
DataIdentifier dat = read.getVariable(var);
+ long dim1 = (dat instanceof IndexedIdentifier) ? ((IndexedIdentifier)dat).getOrigDim1() : dat.getDim1();
+ long dim2 = (dat instanceof IndexedIdentifier) ? ((IndexedIdentifier)dat).getOrigDim2() : dat.getDim2();
DataOp tread = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, DataOpTypes.TRANSIENTREAD,
- dat.getFilename(), dat.getDim1(), dat.getDim2(), dat.getNnz(), blocksize, blocksize);
+ dat.getFilename(), dim1, dim2, dat.getNnz(), blocksize, blocksize);
tread.setRequiresCheckpoint( true );
DataOp twrite = new DataOp(var, DataType.MATRIX, ValueType.DOUBLE, tread, DataOpTypes.TRANSIENTWRITE, null);
- HopRewriteUtils.setOutputParameters(twrite, dat.getDim1(), dat.getDim2(), blocksize, blocksize, dat.getNnz());
+ HopRewriteUtils.setOutputParameters(twrite, dim1, dim2, blocksize, blocksize, dat.getNnz());
hops.add(twrite);
livein.addVariable(var, read.getVariable(var));
liveout.addVariable(var, read.getVariable(var));