You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/02/20 04:06:49 UTC
[3/3] systemml git commit: [SYSTEMML-2157] Fix parfor optimizer side
effect on codegen plans
[SYSTEMML-2157] Fix parfor optimizer side effect on codegen plans
The parfor optimizer applies a variety of rewrites some which require
the recompilation of the parfor body program or parts of it. Most of
these recompilations leverage the recompiler and thus work properly with
codegen. However, there are two rewrites (removal of branches and the
injection of spark checkpoints) that need to recreate not just
instructions but a partial runtime program. This code path was not
integrated with codegen yet and hence led to lost fusion plans whenever
these rewrites triggered (e.g., for Kmeans over a specific range of data
sizes). This patch fixes the underlying code path for partial program
recompilations, which ensures robustness independent of concrete
instances of rewrites. We also modified the Kmeans tests to check for
properly compiled row-wise templates in the inner loop.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/2c9418c3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/2c9418c3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/2c9418c3
Branch: refs/heads/master
Commit: 2c9418c3ec6d81dbb0b11c05e56d2672f1992edd
Parents: b0fff8c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Feb 19 20:06:10 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Feb 19 20:06:46 2018 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/parser/DMLTranslator.java | 6 ++++++
.../controlprogram/parfor/opt/ProgramRecompiler.java | 9 +++++++++
.../integration/functions/codegenalg/AlgorithmKMeans.java | 3 ++-
3 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 8437974..63c896c 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -308,6 +308,12 @@ public class DMLTranslator
SpoofCompiler.generateCode(rtprog);
}
+ public void codgenHopsDAG(ProgramBlock pb)
+ throws HopsException, DMLRuntimeException, LopsException, IOException
+ {
+ SpoofCompiler.generateCodeFromProgramBlock(pb);
+ }
+
public void constructLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException {
// for each namespace, handle function program blocks handle function
for( String namespaceKey : dmlp.getNamespaces().keySet() )
http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index d16d708..3bb0c93 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -28,6 +28,8 @@ import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.IndexingOp;
import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.codegen.SpoofCompiler;
+import org.apache.sysml.hops.codegen.SpoofCompiler.IntegrationType;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopProperties;
@@ -74,6 +76,13 @@ public class ProgramRecompiler
ret.add(dmlt.createRuntimeProgramBlock(rtprog, sb, config));
}
+ //enhance runtime program by automatic operator fusion
+ if( ConfigurationManager.isCodegenEnabled()
+ && SpoofCompiler.INTEGRATION==IntegrationType.RUNTIME ) {
+ for( ProgramBlock pb : ret )
+ dmlt.codgenHopsDAG(pb);
+ }
+
return ret;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java
index c131ea8..f04d684 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java
@@ -172,7 +172,8 @@ public class AlgorithmKMeans extends AutomatedTestBase
runTest(true, false, null, -1);
- Assert.assertTrue(heavyHittersContainsSubString("spoof") || heavyHittersContainsSubString("sp_spoof"));
+ Assert.assertTrue(heavyHittersContainsSubString("spoofCell") || heavyHittersContainsSubString("sp_spoofCell"));
+ Assert.assertTrue(heavyHittersContainsSubString("spoofRA") || heavyHittersContainsSubString("sp_spoofRA"));
}
finally {
rtplatform = platformOld;