You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/11/19 21:47:19 UTC
[37/50] [abbrv] incubator-systemml git commit: Improved size
inference on dynamic recompilation (update awareness)
Improved size inference on dynamic recompilation (update awareness)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4d0378b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4d0378b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4d0378b6
Branch: refs/heads/master
Commit: 4d0378b633726dd450bd2c062b4e6d2dc7671178
Parents: 680940a
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Nov 5 20:38:12 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Nov 5 20:38:12 2015 -0800
----------------------------------------------------------------------
.../com/ibm/bi/dml/hops/recompile/Recompiler.java | 18 ++++++++++++++++--
.../IPAAssignConstantPropagationTest.java | 3 +--
.../recompile/IPAConstantPropagationTest.java | 3 +--
3 files changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4d0378b6/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java b/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
index 0705459..b8e894e 100644
--- a/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
+++ b/src/main/java/com/ibm/bi/dml/hops/recompile/Recompiler.java
@@ -1628,10 +1628,13 @@ public class Recompiler
if( hop.getVisited() == VisitStatus.DONE )
return;
+ //recursively process children
if( hop.getInput() != null )
for( Hop c : hop.getInput() )
rUpdateStatistics(c, vars);
+ boolean updatedSizeExpr = false;
+
//update statistics for transient reads according to current statistics
//(with awareness not to override persistent reads to an existing name)
if( hop instanceof DataOp
@@ -1668,14 +1671,17 @@ public class Recompiler
if ( d.getOp() == DataGenMethod.RAND || d.getOp()==DataGenMethod.SINIT
|| d.getOp() == DataGenMethod.SAMPLE )
{
+ boolean initUnknown = !d.dimsKnown();
int ix1 = params.get(DataExpression.RAND_ROWS);
int ix2 = params.get(DataExpression.RAND_COLS);
//update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
d.refreshRowsParameterInformation(d.getInput().get(ix1), vars);
d.refreshColsParameterInformation(d.getInput().get(ix2), vars);
+ updatedSizeExpr = initUnknown & d.dimsKnown();
}
else if ( d.getOp() == DataGenMethod.SEQ )
{
+ boolean initUnknown = !d.dimsKnown();
int ix1 = params.get(Statement.SEQ_FROM);
int ix2 = params.get(Statement.SEQ_TO);
int ix3 = params.get(Statement.SEQ_INCR);
@@ -1696,6 +1702,7 @@ public class Recompiler
d.setDim2( 1 );
d.setIncrementValue( incr );
}
+ updatedSizeExpr = initUnknown & d.dimsKnown();
}
else {
throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
@@ -1706,8 +1713,10 @@ public class Recompiler
&& ((ReorgOp)(hop)).getOp()==Hop.ReOrgOp.RESHAPE )
{
ReorgOp d = (ReorgOp) hop;
+ boolean initUnknown = !d.dimsKnown();
d.refreshRowsParameterInformation(d.getInput().get(1), vars);
d.refreshColsParameterInformation(d.getInput().get(2), vars);
+ updatedSizeExpr = initUnknown & d.dimsKnown();
}
//update size expression for indexing according to symbol table entries
else if( hop instanceof IndexingOp )
@@ -1717,6 +1726,7 @@ public class Recompiler
Hop input3 = iop.getInput().get(2); //inpRowU
Hop input4 = iop.getInput().get(3); //inpColL
Hop input5 = iop.getInput().get(4); //inpColU
+ boolean initUnknown = !iop.dimsKnown();
double rl = iop.computeBoundsInformation(input2, vars);
double ru = iop.computeBoundsInformation(input3, vars);
double cl = iop.computeBoundsInformation(input4, vars);
@@ -1725,10 +1735,14 @@ public class Recompiler
iop.setDim1( (long)(ru-rl+1) );
if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE )
iop.setDim2( (long)(cu-cl+1) );
+ updatedSizeExpr = initUnknown & iop.dimsKnown();
}
- //propagate statistics along inner nodes of DAG
- hop.refreshSizeInformation();
+ //propagate statistics along inner nodes of DAG,
+ //without overwriting inferred size expressions
+ if( !updatedSizeExpr ) {
+ hop.refreshSizeInformation();
+ }
hop.setVisited(VisitStatus.DONE);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4d0378b6/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
index c4e9fce..c3b9cb3 100644
--- a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
+++ b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAAssignConstantPropagationTest.java
@@ -109,8 +109,7 @@ public class IPAAssignConstantPropagationTest extends AutomatedTestBase
TestUtils.compareMatrices(dmlfile, rfile, 0, "Stat-DML", "Stat-R");
//check expected number of compiled and executed MR jobs
- //FIXME int expectedNumCompiled = ( branchRemoval && IPA ) ? 0 : 1; //rand
- int expectedNumCompiled = 1;
+ int expectedNumCompiled = ( branchRemoval && IPA ) ? 0 : 1; //rand
int expectedNumExecuted = 0;
checkNumCompiledMRJobs(expectedNumCompiled);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4d0378b6/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAConstantPropagationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAConstantPropagationTest.java b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAConstantPropagationTest.java
index 37980dd..4359e04 100644
--- a/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAConstantPropagationTest.java
+++ b/src/test/java/com/ibm/bi/dml/test/integration/functions/recompile/IPAConstantPropagationTest.java
@@ -135,8 +135,7 @@ public class IPAConstantPropagationTest extends AutomatedTestBase
TestUtils.compareMatrices(dmlfile, rfile, 0, "Stat-DML", "Stat-R");
//check expected number of compiled and executed MR jobs
- //FIXME int expectedNumCompiled = ( branchRemoval && IPA && !update ) ? 0 : 1; //rand
- int expectedNumCompiled = 1;
+ int expectedNumCompiled = ( branchRemoval && IPA && !update ) ? 0 : 1; //rand
int expectedNumExecuted = 0;
checkNumCompiledMRJobs(expectedNumCompiled);