You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/24 00:17:36 UTC
[1/2] systemml git commit: [SYSTEMML-2341] Improved recompilation of
right indexing and reshape
Repository: systemml
Updated Branches:
refs/heads/master 72454cdaa -> a13632d45
[SYSTEMML-2341] Improved recompilation of right indexing and reshape
This patch improves special cases of size updates during dynamic
recompilation and thus, also during intra- and inter-procedural analysis
(IPA). Specifically, we now handle the size reset, recursive DAG
propagation, and update with symbol table statistics in a consistent
manner.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/fc5039ed
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/fc5039ed
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/fc5039ed
Branch: refs/heads/master
Commit: fc5039ed7b85949285c9c450dc2635e09214b42d
Parents: 72454cd
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 23 17:10:04 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed May 23 17:10:04 2018 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 4 +-
.../apache/sysml/hops/recompile/Recompiler.java | 74 ++++++++------------
.../recompile/FunctionRecompileTest.java | 6 +-
3 files changed, 36 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 7b0ac5b..de84088 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1720,11 +1720,11 @@ public abstract class Hop implements ParseInfo
return ret;
}
- public double computeBoundsInformation( Hop input, LocalVariableMap vars ) {
+ public final double computeBoundsInformation( Hop input, LocalVariableMap vars ) {
return computeBoundsInformation(input, vars, new HashMap<Long, Double>());
}
- public double computeBoundsInformation( Hop input, LocalVariableMap vars, HashMap<Long, Double> memo ) {
+ public final double computeBoundsInformation( Hop input, LocalVariableMap vars, HashMap<Long, Double> memo ) {
double ret = Double.MAX_VALUE;
try {
ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, memo, vars);
http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index fe69b49..0364724 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -46,12 +46,12 @@ import org.apache.sysml.hops.Hop.DataGenMethod;
import org.apache.sysml.hops.Hop.DataOpTypes;
import org.apache.sysml.hops.Hop.FileFormatTypes;
import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.ReOrgOp;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.IndexingOp;
import org.apache.sysml.hops.LiteralOp;
import org.apache.sysml.hops.MemoTable;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.hops.ReorgOp;
import org.apache.sysml.hops.UnaryOp;
import org.apache.sysml.hops.codegen.SpoofCompiler;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
@@ -1282,17 +1282,12 @@ public class Recompiler
for( Hop c : hop.getInput() )
rUpdateStatistics(c, vars);
- boolean updatedSizeExpr = false;
-
//update statistics for transient reads according to current statistics
//(with awareness not to override persistent reads to an existing name)
- if( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType() != DataOpTypes.PERSISTENTREAD )
- {
+ if( HopRewriteUtils.isData(hop, DataOpTypes.TRANSIENTREAD) ) {
DataOp d = (DataOp) hop;
String varName = d.getName();
- if( vars.keySet().contains( varName ) )
- {
+ if( vars.keySet().contains( varName ) ) {
Data dat = vars.get(varName);
if( dat instanceof MatrixObject ) {
MatrixObject mo = (MatrixObject) dat;
@@ -1308,10 +1303,9 @@ public class Recompiler
}
}
//special case for persistent reads with unknown size (read-after-write)
- else if( hop instanceof DataOp
- && ((DataOp)hop).getDataOpType() == DataOpTypes.PERSISTENTREAD
- && !hop.dimsKnown() && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.CSV
- && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA) )
+ else if( HopRewriteUtils.isData(hop, DataOpTypes.PERSISTENTREAD)
+ && !hop.dimsKnown() && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.CSV
+ && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA) )
{
//update hop with read meta data
DataOp dop = (DataOp) hop;
@@ -1332,7 +1326,8 @@ public class Recompiler
HashMap<Long, Long> memo = new HashMap<>();
d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo);
d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo);
- updatedSizeExpr = initUnknown & d.dimsKnown();
+ if( !(initUnknown & d.dimsKnown()) )
+ d.refreshSizeInformation();
}
else if ( d.getOp() == DataGenMethod.SEQ )
{
@@ -1355,47 +1350,40 @@ public class Recompiler
d.setDim2( 1 );
d.setIncrementValue( incr );
}
- updatedSizeExpr = initUnknown & d.dimsKnown();
+ if( !(initUnknown & d.dimsKnown()) )
+ d.refreshSizeInformation();
}
else {
throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
}
}
//update size expression for reshape according to symbol table entries
- else if ( hop instanceof ReorgOp
- && ((ReorgOp)(hop)).getOp()==Hop.ReOrgOp.RESHAPE )
- {
- ReorgOp d = (ReorgOp) hop;
- boolean initUnknown = !d.dimsKnown();
- HashMap<Long, Long> memo = new HashMap<>();
- d.refreshRowsParameterInformation(d.getInput().get(1), vars, memo);
- d.refreshColsParameterInformation(d.getInput().get(2), vars, memo);
- updatedSizeExpr = initUnknown & d.dimsKnown();
+ else if( HopRewriteUtils.isReorg(hop, ReOrgOp.RESHAPE) ) {
+ hop.refreshSizeInformation(); //update incl reset
+ if( !hop.dimsKnown() ) {
+ HashMap<Long, Long> memo = new HashMap<>();
+ hop.refreshRowsParameterInformation(hop.getInput().get(1), vars, memo);
+ hop.refreshColsParameterInformation(hop.getInput().get(2), vars, memo);
+ }
}
//update size expression for indexing according to symbol table entries
- else if( hop instanceof IndexingOp && hop.getDataType()!=DataType.LIST )
- {
- IndexingOp iop = (IndexingOp)hop;
- Hop input2 = iop.getInput().get(1); //inpRowL
- Hop input3 = iop.getInput().get(2); //inpRowU
- Hop input4 = iop.getInput().get(3); //inpColL
- Hop input5 = iop.getInput().get(4); //inpColU
- boolean initUnknown = !iop.dimsKnown();
- HashMap<Long, Double> memo = new HashMap<>();
- double rl = iop.computeBoundsInformation(input2, vars, memo);
- double ru = iop.computeBoundsInformation(input3, vars, memo);
- double cl = iop.computeBoundsInformation(input4, vars, memo);
- double cu = iop.computeBoundsInformation(input5, vars, memo);
- if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE )
- iop.setDim1( (long)(ru-rl+1) );
- if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE )
- iop.setDim2( (long)(cu-cl+1) );
- updatedSizeExpr = initUnknown & iop.dimsKnown();
+ else if( hop instanceof IndexingOp && hop.getDataType()!=DataType.LIST ) {
+ hop.refreshSizeInformation(); //update, incl reset
+ if( !hop.dimsKnown() ) {
+ HashMap<Long, Double> memo = new HashMap<>();
+ double rl = hop.computeBoundsInformation(hop.getInput().get(1), vars, memo);
+ double ru = hop.computeBoundsInformation(hop.getInput().get(2), vars, memo);
+ double cl = hop.computeBoundsInformation(hop.getInput().get(3), vars, memo);
+ double cu = hop.computeBoundsInformation(hop.getInput().get(4), vars, memo);
+ if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE )
+ hop.setDim1( (long)(ru-rl+1) );
+ if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE )
+ hop.setDim2( (long)(cu-cl+1) );
+ }
}
-
+ else {
//propagate statistics along inner nodes of DAG,
//without overwriting inferred size expressions
- if( !updatedSizeExpr ) {
hop.refreshSizeInformation();
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
index 55a2d5b..60f263f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
@@ -103,15 +103,15 @@ public class FunctionRecompileTest extends AutomatedTestBase
//note: change from previous version due to fix in op selection (unknown size XtX and mapmult)
//CHECK compiled MR jobs
int expectNumCompiled = -1;
- if( IPA ) expectNumCompiled = 4; //reblock TODO investigate 1-4 recompile side effect
- else expectNumCompiled = 5;//reblock, GMR,GMR,GMR,GMR (last two should piggybacked)
+ if( IPA ) expectNumCompiled = 1; //reblock
+ else expectNumCompiled = 5; //reblock, GMR,GMR,GMR,GMR (last two should piggybacked)
Assert.assertEquals("Unexpected number of compiled MR jobs.",
expectNumCompiled, Statistics.getNoOfCompiledMRJobs());
//CHECK executed MR jobs
int expectNumExecuted = -1;
if( recompile ) expectNumExecuted = 0;
- else if( IPA ) expectNumExecuted = 31; //reblock TODO investigate 1-31 recompile side effect
+ else if( IPA ) expectNumExecuted = 1; //reblock
else expectNumExecuted = 41; //reblock, 10*(GMR,GMR,GMR, GMR) (last two should piggybacked)
Assert.assertEquals("Unexpected number of executed MR jobs.",
expectNumExecuted, Statistics.getNoOfExecutedMRJobs());
[2/2] systemml git commit: [MINOR] Add more test suites to
integration tests (estim, ext, frame)
Posted by mb...@apache.org.
[MINOR] Add more test suites to integration tests (estim, ext, frame)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a13632d4
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a13632d4
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a13632d4
Branch: refs/heads/master
Commit: a13632d4582b0d74c5898fe67790a1052ba11c2f
Parents: fc5039e
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 23 17:17:45 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed May 23 17:17:45 2018 -0700
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/a13632d4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 653fa09..40b3523 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,9 @@
<!-- <include>**/integration/applications/**/*Suite.java</include> -->
<!-- <include>**/integration/conversion/*Suite.java</include> -->
<!-- <include>**/integration/functions/data/*Suite.java</include> -->
+ <include>**/integration/functions/estim/*Suite.java</include>
+ <include>**/integration/functions/external/*Suite.java</include>
+ <include>**/integration/functions/frame/*Suite.java</include>
<include>**/integration/functions/jmlc/*Suite.java</include>
<include>**/integration/functions/misc/*Suite.java</include>
<include>**/integration/functions/sparse/*Suite.java</include>