You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/02/12 19:32:15 UTC
[1/5] incubator-systemml git commit: [MINOR] Cleanups (missing
imports, unnecessary tags, fix max blocks)
Repository: incubator-systemml
Updated Branches:
refs/heads/master ca4e2600e -> 100075046
[MINOR] Cleanups (missing imports, unnecessary tags, fix max blocks)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/696d10b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/696d10b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/696d10b5
Branch: refs/heads/master
Commit: 696d10b5ae05704918c4fca76ef6d3fb635ec228
Parents: ca4e260
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 01:14:16 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 01:14:16 2017 +0100
----------------------------------------------------------------------
.../controlprogram/context/SparkExecutionContext.java | 2 +-
.../sysml/runtime/instructions/spark/SPInstruction.java | 2 --
.../apache/sysml/runtime/matrix/data/LibMatrixCUDA.java | 11 ++---------
.../test/integration/functions/mlcontext/GNMFTest.java | 1 -
4 files changed, 3 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/696d10b5/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index bbc9fb3..d1e521a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -1292,7 +1292,7 @@ public class SparkExecutionContext extends ExecutionContext
// The most expensive operation here is rdd.toDebugString() which can be a major hit because
// of unrolling lazy evaluation of Spark. Hence, it is guarded against it along with flag 'PRINT_EXPLAIN_WITH_LINEAGE' which is
// enabled only through MLContext. This way, it doesnot affect our performance evaluation through non-MLContext path
- @SuppressWarnings("deprecation")
+ @SuppressWarnings("unused")
private void setLineageInfoForExplain(SPInstruction inst,
JavaPairRDD<?, ?> out,
JavaPairRDD<?, ?> in1, String in1Name,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/696d10b5/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
index 5660cb3..8f866af 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
@@ -74,7 +74,6 @@ public abstract class SPInstruction extends Instruction
return getOpcode();
}
- @SuppressWarnings("deprecation")
@Override
public Instruction preprocessInstruction(ExecutionContext ec)
throws DMLRuntimeException
@@ -97,7 +96,6 @@ public abstract class SPInstruction extends Instruction
public abstract void processInstruction(ExecutionContext ec)
throws DMLRuntimeException;
- @SuppressWarnings("deprecation")
@Override
public void postprocessInstruction(ExecutionContext ec)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/696d10b5/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
index 9d3220b..c10d0bf 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java
@@ -84,8 +84,6 @@ import jcuda.jcudnn.cudnnTensorDescriptor;
import jcuda.jcusparse.JCusparse;
import jcuda.jcusparse.cusparseHandle;
-import java.util.Vector;
-
//FIXME move could to respective instructions, this is not a block library
public class LibMatrixCUDA {
@@ -992,12 +990,6 @@ public class LibMatrixCUDA {
//**************** UNARY AGGREGATE Functions ************************/
//********************************************************************/
- /**
- * Direction of reduction for aggregate binary operations
- */
- private enum ReductionDirection{
- ALL, ROW, COL, DIAG;
- };
/**
* Entry point to perform Unary aggregate operations on the GPU.
@@ -1436,7 +1428,7 @@ public class LibMatrixCUDA {
final int MAX_BLOCKS = getMaxBlocks();
final int WARP_SIZE = getWarpSize();
int threads = Math.min(cols, MAX_THREADS);
- int blocks = cols/MAX_THREADS;
+ int blocks = Math.min(cols/MAX_THREADS, MAX_BLOCKS);
if (cols % MAX_THREADS != 0) blocks++;
int sharedMemSize = threads * Sizeof.DOUBLE;
if (threads <= WARP_SIZE){
@@ -2232,6 +2224,7 @@ public class LibMatrixCUDA {
* @param rlen row length
* @param clen column length
*/
+ @SuppressWarnings("unused")
private static void debugPrintMatrix(Pointer in, int rlen, int clen){
double[] data = new double[rlen * clen];
cudaMemcpy(Pointer.to(data), in, rlen*clen*Sizeof.DOUBLE, cudaMemcpyDeviceToHost);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/696d10b5/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
index 99ab53b..a826443 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
@@ -64,7 +64,6 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-@SuppressWarnings("deprecation")
@RunWith(value = Parameterized.class)
public class GNMFTest extends AutomatedTestBase
{
[4/5] incubator-systemml git commit: [SYSTEMML-1254] New sum-product
rewrites (agg pushdown), stratstats
Posted by mb...@apache.org.
[SYSTEMML-1254] New sum-product rewrites (agg pushdown), stratstats
In the spirit of our SPOOF compiler framework and the existing
sum(X%*%Y) rewrite, this patch adds the following two sum-product
rewrites (where the first applies multiple times in stratstats):
* colSums(X %*% Y) -> colsSums(X) %*% Y
* rowSums(X %*% Y) -> X %*% rowSums(Y)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b3ba9916
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b3ba9916
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b3ba9916
Branch: refs/heads/master
Commit: b3ba991604cf79c5e3e2c0992fe2439ae47ce023
Parents: d3e617b
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 08:29:36 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 09:42:03 2017 +0100
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationDynamic.java | 54 +++++++++++---------
1 file changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b3ba9916/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index e8e3862..6ffcbd5 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -2547,11 +2547,12 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
private Hop simplifySumMatrixMult(Hop parent, Hop hi, int pos)
{
- //sum(A%*%B) -> sum(t(colSums(A))*rowSums(B))
- //if not dot product, not applied since aggregate removed
- //if sum not the only consumer, not applied to prevent redundancy
+ //sum(A%*%B) -> sum(t(colSums(A))*rowSums(B)), later rewritten to dot-product
+ //colSums(A%*%B) -> colSums(A)%*%B
+ //rowSums(A%*%B) -> A%*%rowSums(B)
+ //-- if not dot product, not applied since aggregate removed
+ //-- if sum not the only consumer, not applied to prevent redundancy
if( hi instanceof AggUnaryOp && ((AggUnaryOp)hi).getOp()==AggOp.SUM //sum
- && ((AggUnaryOp)hi).getDirection() == Direction.RowCol //full aggregate
&& hi.getInput().get(0) instanceof AggBinaryOp //A%*%B
&& (hi.getInput().get(0).getDim1()>1 || hi.getInput().get(0).getDim2()>1) //not dot product
&& hi.getInput().get(0).getParent().size()==1 ) //not multiple consumers of matrix mult
@@ -2560,34 +2561,39 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
Hop left = hi2.getInput().get(0);
Hop right = hi2.getInput().get(1);
- //remove link from parent to diag
+ //remove link from parent to matrix mult
HopRewriteUtils.removeChildReference(hi, hi2);
//create new operators
- AggUnaryOp colSum = new AggUnaryOp(left.getName(), left.getDataType(), left.getValueType(), AggOp.SUM, Direction.Col, left);
- colSum.setRowsInBlock(left.getRowsInBlock());
- colSum.setColsInBlock(left.getColsInBlock());
- colSum.refreshSizeInformation();
- ReorgOp trans = HopRewriteUtils.createTranspose(colSum);
- AggUnaryOp rowSum = new AggUnaryOp(right.getName(), right.getDataType(), right.getValueType(), AggOp.SUM, Direction.Row, right);
- rowSum.setRowsInBlock(right.getRowsInBlock());
- rowSum.setColsInBlock(right.getColsInBlock());
- rowSum.refreshSizeInformation();
- BinaryOp mult = new BinaryOp(right.getName(), right.getDataType(), right.getValueType(), OpOp2.MULT, trans, rowSum);
- mult.setRowsInBlock(right.getRowsInBlock());
- mult.setColsInBlock(right.getColsInBlock());
- mult.refreshSizeInformation();
-
+ Hop root = null;
+ //pattern: sum(A%*%B) -> sum(t(colSums(A))*rowSums(B)), later rewritten to dot-product
+ if( ((AggUnaryOp)hi).getDirection() == Direction.RowCol ) {
+ AggUnaryOp colSum = HopRewriteUtils.createAggUnaryOp(left, AggOp.SUM, Direction.Col);
+ ReorgOp trans = HopRewriteUtils.createTranspose(colSum);
+ AggUnaryOp rowSum = HopRewriteUtils.createAggUnaryOp(right, AggOp.SUM, Direction.Row);
+ root = HopRewriteUtils.createBinary(trans, rowSum, OpOp2.MULT);
+ LOG.debug("Applied simplifySumMatrixMult RC.");
+ }
+ //colSums(A%*%B) -> colSums(A)%*%B
+ else if( ((AggUnaryOp)hi).getDirection() == Direction.Col ) {
+ AggUnaryOp colSum = HopRewriteUtils.createAggUnaryOp(left, AggOp.SUM, Direction.Col);
+ root = HopRewriteUtils.createMatrixMultiply(colSum, right);
+ LOG.debug("Applied simplifySumMatrixMult C.");
+ }
+ //rowSums(A%*%B) -> A%*%rowSums(B)
+ else if( ((AggUnaryOp)hi).getDirection() == Direction.Row ) {
+ AggUnaryOp rowSum = HopRewriteUtils.createAggUnaryOp(right, AggOp.SUM, Direction.Row);
+ root = HopRewriteUtils.createMatrixMultiply(left, rowSum);
+ LOG.debug("Applied simplifySumMatrixMult R.");
+ }
//rehang new subdag under current node (keep hi intact)
- HopRewriteUtils.addChildReference(hi, mult, 0);
+ HopRewriteUtils.addChildReference(hi, root, 0);
hi.refreshSizeInformation();
-
+
//cleanup if only consumer of intermediate
if( hi2.getParent().isEmpty() )
- HopRewriteUtils.removeAllChildReferences( hi2 );
-
- LOG.debug("Applied simplifySumMatrixMult.");
+ HopRewriteUtils.removeAllChildReferences( hi2 );
}
return hi;
[3/5] incubator-systemml git commit: [SYSTEMML-1253] Selective wdivmm
rewrite (dense factors), stratstats
Posted by mb...@apache.org.
[SYSTEMML-1253] Selective wdivmm rewrite (dense factors), stratstats
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d3e617b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d3e617b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d3e617b0
Branch: refs/heads/master
Commit: d3e617b0c9f65cc1b80e884e8df6c7d3668332b0
Parents: 0a3d248
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 07:44:03 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 09:41:48 2017 +0100
----------------------------------------------------------------------
.../sysml/hops/rewrite/HopRewriteUtils.java | 6 +++++
.../RewriteAlgebraicSimplificationDynamic.java | 26 +++++++++++---------
2 files changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d3e617b0/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 8ccb531..50501dc 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -60,6 +60,7 @@ import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.UtilFunctions;
public class HopRewriteUtils
@@ -672,6 +673,11 @@ public class HopRewriteUtils
&& hop.getInput().get(1).getDim1() < hop.getInput().get(1).getDim2();
}
+ public static boolean isSparse( Hop hop ) {
+ return hop.dimsKnown(true) //dims and nnz known
+ && MatrixBlock.evalSparseFormatInMemory(hop.getDim1(), hop.getDim2(), hop.getNnz());
+ }
+
public static boolean isEqualValue( LiteralOp hop1, LiteralOp hop2 )
throws HopsException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d3e617b0/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index ed89a05..e8e3862 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -1972,18 +1972,22 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
Hop U = hi.getInput().get(1).getInput().get(0);
Hop V = hi.getInput().get(1).getInput().get(1);
- if( !HopRewriteUtils.isTransposeOperation(V) )
- V = HopRewriteUtils.createTranspose(V);
- else
- V = V.getInput().get(0);
+ //for this basic pattern, we're more conservative and only apply wdivmm if
+ //the factors are not known to be sparse
+ if( !HopRewriteUtils.isSparse(U) && !HopRewriteUtils.isSparse(V) ) {
+ if( !HopRewriteUtils.isTransposeOperation(V) )
+ V = HopRewriteUtils.createTranspose(V);
+ else
+ V = V.getInput().get(0);
- hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
- OpOp4.WDIVMM, W, U, V, new LiteralOp(-1), 0, true, false);
- HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
- hnew.refreshSizeInformation();
-
- appliedPattern = true;
- LOG.debug("Applied simplifyWeightedDivMM7 (line "+hi.getBeginLine()+")");
+ hnew = new QuaternaryOp(hi.getName(), DataType.MATRIX, ValueType.DOUBLE,
+ OpOp4.WDIVMM, W, U, V, new LiteralOp(-1), 0, true, false);
+ HopRewriteUtils.setOutputBlocksizes(hnew, W.getRowsInBlock(), W.getColsInBlock());
+ hnew.refreshSizeInformation();
+
+ appliedPattern = true;
+ LOG.debug("Applied simplifyWeightedDivMM7 (line "+hi.getBeginLine()+")");
+ }
}
//relink new hop into original position
[5/5] incubator-systemml git commit: [HOTFIX] Fix csv frame readers
(robust size computation), tests
Posted by mb...@apache.org.
[HOTFIX] Fix csv frame readers (robust size computation), tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/10007504
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/10007504
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/10007504
Branch: refs/heads/master
Commit: 1000750464d470a7b56db71d896c6ca25e11e0bd
Parents: b3ba991
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 20:15:00 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 20:15:00 2017 +0100
----------------------------------------------------------------------
.../sysml/runtime/io/IOUtilFunctions.java | 10 ++--
.../TransformCSVFrameEncodeReadTest.java | 48 ++++++++++++++++----
.../transform/TransformCSVFrameEncodeRead.dml | 3 +-
3 files changed, 48 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 3f0ea56..9d3b0ab 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -382,19 +382,21 @@ public class IOUtilFunctions
{
LongWritable key = new LongWritable();
Text value = new Text();
- int ncol = -1;
+ int ncol = -1;
for( int i=0; i<splits.length && ncol<=0; i++ ) {
RecordReader<LongWritable, Text> reader =
informat.getRecordReader(splits[i], job, Reporter.NULL);
try {
if( reader.next(key, value) ) {
+ boolean hasValue = true;
if( value.toString().startsWith(TfUtils.TXMTD_MVPREFIX) )
- reader.next(key, value);
+ hasValue = reader.next(key, value);
if( value.toString().startsWith(TfUtils.TXMTD_NDPREFIX) )
- reader.next(key, value);
+ hasValue = reader.next(key, value);
String row = value.toString().trim();
- if( !row.isEmpty() )
+ if( hasValue && !row.isEmpty() ) {
ncol = IOUtilFunctions.countTokensCSV(row, delim);
+ }
}
}
finally {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
index b28c2df..b35f2ac 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java
@@ -51,33 +51,64 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
@Test
public void testFrameReadMetaSingleNodeCSV() {
- runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, false);
}
@Test
public void testFrameReadMetaSparkCSV() {
- runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, false);
}
@Test
public void testFrameReadMetaHybridCSV() {
- runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false);
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, false);
}
@Test
public void testFrameParReadMetaSingleNodeCSV() {
- runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, true);
}
@Test
public void testFrameParReadMetaSparkCSV() {
- runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, true);
}
@Test
public void testFrameParReadMetaHybridCSV() {
- runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true);
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, true);
}
+
+ @Test
+ public void testFrameReadSubMetaSingleNodeCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameReadSubMetaSparkCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameReadSubMetaHybridCSV() {
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, false);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaSingleNodeCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, true);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaSparkCSV() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, true);
+ }
+
+ @Test
+ public void testFrameParReadSubMetaHybridCSV() {
+ runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, true);
+ }
+
/**
*
@@ -85,7 +116,7 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
* @param ofmt
* @param dataset
*/
- private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean parRead )
+ private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean subset, boolean parRead )
{
//set runtime platform
RUNTIME_PLATFORM rtold = rtplatform;
@@ -104,9 +135,10 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase
getAndLoadTestConfiguration(TEST_NAME1);
String HOME = SCRIPT_DIR + TEST_DIR;
+ int nrows = subset ? 4 : 13;
fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
programArgs = new String[]{"-explain", "-stats","-args",
- HOME + "input/" + DATASET, output("R") };
+ HOME + "input/" + DATASET, String.valueOf(nrows), output("R") };
OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true;
runTest(true, false, null, -1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
index 9da935f..d75cf8a 100644
--- a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
+++ b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml
@@ -24,6 +24,7 @@ jspec = "{\"ids\": true, \"recode\": [1,2,3]}";
[X, M] = transformencode(target=F1, spec=jspec);
+M = M[1:$2,]
print(toString(M))
-write(M, $2, format="csv");
+write(M, $3, format="csv");
[2/5] incubator-systemml git commit: [SYSTEMML-1251] Fix
recompile-once recursive functions, cleanup, tests
Posted by mb...@apache.org.
[SYSTEMML-1251] Fix recompile-once recursive functions, cleanup, tests
This patch fixes our inter-procedural analysis, disallowing to mark
directly or indirectly recursive functions for recompile-once because
recompile-once can lead to OOMs and even incorrect results in the
context of recursion. Recompile-once functions are recompiled on entry
with the given input statistics in order to avoid repeated recompilation
in loops, which is often unnecessary knowing the size of function
inputs. However, with recursive functions it is not that easy. A subcall
recompiles the plan again, also modifying the remaining plan for the
calling function - if both recursion levels work on different sizes or
compile literals into the plan, the consequences are disastrous.
Since multiple places such as EXPLAIN and IPA require information about
the function call graph, this patch introduces a clean abstraction, the
FunctionCallGraph, that captures this information once and simplifies
related compiler passes.
Furthermore, this patch also includes a fix for our -stats tool to
properly reset the number and time of function recompilations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/0a3d2481
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/0a3d2481
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/0a3d2481
Branch: refs/heads/master
Commit: 0a3d24815258744169367acd8f0287761b5d20de
Parents: 696d10b
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Feb 12 06:27:58 2017 +0100
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Feb 12 06:27:58 2017 +0100
----------------------------------------------------------------------
.../sysml/hops/ipa/FunctionCallGraph.java | 247 +++++++++++++++++++
.../sysml/hops/ipa/InterProceduralAnalysis.java | 38 +--
.../org/apache/sysml/parser/DMLProgram.java | 8 +
.../java/org/apache/sysml/utils/Explain.java | 131 +++-------
.../java/org/apache/sysml/utils/Statistics.java | 3 +
.../RecursiveFunctionRecompileTest.java | 149 +++++++++++
.../recompile/recursive_func_direct.dml | 38 +++
.../recompile/recursive_func_indirect.dml | 46 ++++
.../recompile/recursive_func_indirect2.dml | 55 +++++
.../functions/recompile/recursive_func_none.dml | 39 +++
.../functions/recompile/ZPackageSuite.java | 1 +
11 files changed, 645 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java b/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java
new file mode 100644
index 0000000..eed6531
--- /dev/null
+++ b/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.hops.ipa;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Stack;
+
+import org.apache.sysml.hops.FunctionOp;
+import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.HopsException;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+
+public class FunctionCallGraph
+{
+ //internal function key for main program (underscore
+ //prevents any conflicts with user-defined functions)
+ private static final String MAIN_FUNCTION_KEY = "_main";
+
+ //unrolled function call graph, in call direction
+ //(mapping from function keys to called function keys)
+ private final HashMap<String, HashSet<String>> _fGraph;
+
+ //subset of direct or indirect recursive functions
+ private final HashSet<String> _fRecursive;
+
+ /**
+ * Constructs the function call graph for all functions
+ * reachable from the main program.
+ *
+ * @param prog dml program of given script
+ */
+ public FunctionCallGraph(DMLProgram prog) {
+ _fGraph = new HashMap<String, HashSet<String>>();
+ _fRecursive = new HashSet<String>();
+
+ constructFunctionCallGraph(prog);
+ }
+
+ /**
+ * Returns all functions called from the given function.
+ *
+ * @param fnamespace function namespace
+ * @param fname function name
+ * @return list of function keys (namespace and name)
+ */
+ public Collection<String> getCalledFunctions(String fnamespace, String fname) {
+ return getCalledFunctions(
+ DMLProgram.constructFunctionKey(fnamespace, fname));
+ }
+
+ /**
+ * Returns all functions called from the given function.
+ *
+ * @param fkey function key of calling function, null indicates the main program
+ * @return list of function keys (namespace and name)
+ */
+ public Collection<String> getCalledFunctions(String fkey) {
+ String lfkey = (fkey == null) ? MAIN_FUNCTION_KEY : fkey;
+ return _fGraph.get(lfkey);
+ }
+
+ /**
+ * Indicates if the given function is either directly or indirectly recursive.
+ * An example of an indirect recursive function is foo2 in the following call
+ * chain: foo1 -> foo2 -> foo1.
+ *
+ * @param fnamespace function namespace
+ * @param fname function name
+ * @return true if the given function is recursive, false otherwise
+ */
+ public boolean isRecursiveFunction(String fnamespace, String fname) {
+ return isRecursiveFunction(
+ DMLProgram.constructFunctionKey(fnamespace, fname));
+ }
+
+ /**
+ * Indicates if the given function is either directly or indirectly recursive.
+ * An example of an indirect recursive function is foo2 in the following call
+ * chain: foo1 -> foo2 -> foo1.
+ *
+ * @param fkey function key of calling function, null indicates the main program
+ * @return true if the given function is recursive, false otherwise
+ */
+ public boolean isRecursiveFunction(String fkey) {
+ String lfkey = (fkey == null) ? MAIN_FUNCTION_KEY : fkey;
+ return _fRecursive.contains(lfkey);
+ }
+
+ /**
+ * Returns all functions that are reachable either directly or indirectly
+ * form the main program, except the main program itself and the given
+ * blacklist of function names.
+ *
+ * @param blacklist list of function keys to exclude
+ * @return list of function keys (namespace and name)
+ */
+ public Collection<String> getReachableFunctions(Collection<String> blacklist) {
+ HashSet<String> ret = new HashSet<String>();
+ for( String tmp : _fGraph.keySet() )
+ if( !blacklist.contains(tmp) && !MAIN_FUNCTION_KEY.equals(tmp) )
+ ret.add(tmp);
+ return ret;
+ }
+
+ /**
+ * Indicates if the given function is reachable either directly or indirectly
+ * from the main program.
+ *
+ * @param fnamespace function namespace
+ * @param fname function name
+ * @return true if the given function is reachable, false otherwise
+ */
+ public boolean isReachableFunction(String fnamespace, String fname) {
+ return isReachableFunction(
+ DMLProgram.constructFunctionKey(fnamespace, fname));
+ }
+
+ /**
+ * Indicates if the given function is reachable either directly or indirectly
+ * from the main program.
+ *
+ * @param fkey function key of calling function, null indicates the main program
+ * @return true if the given function is reachable, false otherwise
+ */
+ public boolean isReachableFunction(String fkey) {
+ String lfkey = (fkey == null) ? MAIN_FUNCTION_KEY : fkey;
+ return _fGraph.containsKey(lfkey);
+ }
+
+ private void constructFunctionCallGraph(DMLProgram prog) {
+ if( !prog.hasFunctionStatementBlocks() )
+ return; //early abort if prog without functions
+
+ try {
+ Stack<String> fstack = new Stack<String>();
+ HashSet<String> lfset = new HashSet<String>();
+ _fGraph.put(MAIN_FUNCTION_KEY, new HashSet<String>());
+ for( StatementBlock sblk : prog.getStatementBlocks() )
+ rConstructFunctionCallGraph(MAIN_FUNCTION_KEY, sblk, fstack, lfset);
+ }
+ catch(HopsException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private void rConstructFunctionCallGraph(String fkey, StatementBlock sb, Stack<String> fstack, HashSet<String> lfset)
+ throws HopsException
+ {
+ if (sb instanceof WhileStatementBlock) {
+ WhileStatement ws = (WhileStatement)sb.getStatement(0);
+ for (StatementBlock current : ws.getBody())
+ rConstructFunctionCallGraph(fkey, current, fstack, lfset);
+ }
+ else if (sb instanceof IfStatementBlock) {
+ IfStatement ifs = (IfStatement) sb.getStatement(0);
+ for (StatementBlock current : ifs.getIfBody())
+ rConstructFunctionCallGraph(fkey, current, fstack, lfset);
+ for (StatementBlock current : ifs.getElseBody())
+ rConstructFunctionCallGraph(fkey, current, fstack, lfset);
+ }
+ else if (sb instanceof ForStatementBlock) {
+ ForStatement fs = (ForStatement)sb.getStatement(0);
+ for (StatementBlock current : fs.getBody())
+ rConstructFunctionCallGraph(fkey, current, fstack, lfset);
+ }
+ else if (sb instanceof FunctionStatementBlock) {
+ FunctionStatement fsb = (FunctionStatement) sb.getStatement(0);
+ for (StatementBlock current : fsb.getBody())
+ rConstructFunctionCallGraph(fkey, current, fstack, lfset);
+ }
+ else {
+ // For generic StatementBlock
+ ArrayList<Hop> hopsDAG = sb.get_hops();
+ if( hopsDAG == null || hopsDAG.isEmpty() )
+ return; //nothing to do
+
+ //function ops can only occur as root nodes of the dag
+ for( Hop h : hopsDAG ) {
+ if( h instanceof FunctionOp ){
+ FunctionOp fop = (FunctionOp) h;
+ String lfkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
+ //prevent redundant call edges
+ if( lfset.contains(lfkey) || fop.getFunctionNamespace().equals(DMLProgram.INTERNAL_NAMESPACE) )
+ continue;
+
+ if( !_fGraph.containsKey(lfkey) )
+ _fGraph.put(lfkey, new HashSet<String>());
+
+ //recursively construct function call dag
+ if( !fstack.contains(lfkey) ) {
+ fstack.push(lfkey);
+ _fGraph.get(fkey).add(lfkey);
+
+ FunctionStatementBlock fsb = sb.getDMLProg()
+ .getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
+ FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
+ for( StatementBlock csb : fs.getBody() )
+ rConstructFunctionCallGraph(lfkey, csb, fstack, new HashSet<String>());
+ fstack.pop();
+ }
+ //recursive function call
+ else {
+ _fGraph.get(fkey).add(lfkey);
+ _fRecursive.add(lfkey);
+
+ //mark indirectly recursive functions as recursive
+ int ix = fstack.indexOf(lfkey);
+ for( int i=ix+1; i<fstack.size(); i++ )
+ _fRecursive.add(fstack.get(i));
+ }
+
+ //mark as visited for current function call context
+ lfset.add( lfkey );
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index 5d94a68..caab391 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
@@ -157,26 +156,25 @@ public class InterProceduralAnalysis
* @throws ParseException if ParseException occurs
* @throws LanguageException if LanguageException occurs
*/
- @SuppressWarnings("unchecked")
public void analyzeProgram( DMLProgram dmlp )
throws HopsException, ParseException, LanguageException
{
+ FunctionCallGraph fgraph = new FunctionCallGraph(dmlp);
+
//step 1: get candidates for statistics propagation into functions (if required)
Map<String, Integer> fcandCounts = new HashMap<String, Integer>();
Map<String, FunctionOp> fcandHops = new HashMap<String, FunctionOp>();
Map<String, Set<Long>> fcandSafeNNZ = new HashMap<String, Set<Long>>();
- Set<String> allFCandKeys = new HashSet<String>();
if( !dmlp.getFunctionStatementBlocks().isEmpty() ) {
for ( StatementBlock sb : dmlp.getStatementBlocks() ) //get candidates (over entire program)
getFunctionCandidatesForStatisticPropagation( sb, fcandCounts, fcandHops );
- allFCandKeys.addAll(fcandCounts.keySet()); //cp before pruning
pruneFunctionCandidatesForStatisticPropagation( fcandCounts, fcandHops );
determineFunctionCandidatesNNZPropagation( fcandHops, fcandSafeNNZ );
DMLTranslator.resetHopsDAGVisitStatus( dmlp );
}
//step 2: get unary dimension-preserving non-candidate functions
- Collection<String> unaryFcandTmp = CollectionUtils.subtract(allFCandKeys, fcandCounts.keySet());
+ Collection<String> unaryFcandTmp = fgraph.getReachableFunctions(fcandCounts.keySet());
HashSet<String> unaryFcands = new HashSet<String>();
if( !unaryFcandTmp.isEmpty() && UNARY_DIMS_PRESERVING_FUNS ) {
for( String tmp : unaryFcandTmp )
@@ -194,12 +192,12 @@ public class InterProceduralAnalysis
//step 4: remove unused functions (e.g., inlined or never called)
if( REMOVE_UNUSED_FUNCTIONS ) {
- removeUnusedFunctions( dmlp, allFCandKeys );
+ removeUnusedFunctions( dmlp, fgraph );
}
//step 5: flag functions with loops for 'recompile-on-entry'
if( FLAG_FUNCTION_RECOMPILE_ONCE ) {
- flagFunctionsForRecompileOnce( dmlp );
+ flagFunctionsForRecompileOnce( dmlp, fgraph );
}
//step 6: set global data flow properties
@@ -871,22 +869,21 @@ public class InterProceduralAnalysis
// REMOVE UNUSED FUNCTIONS
//////
- public void removeUnusedFunctions( DMLProgram dmlp, Set<String> fcandKeys )
+ public void removeUnusedFunctions( DMLProgram dmlp, FunctionCallGraph fgraph )
throws LanguageException
{
Set<String> fnamespaces = dmlp.getNamespaces().keySet();
- for( String fnspace : fnamespaces )
- {
+ for( String fnspace : fnamespaces ) {
HashMap<String, FunctionStatementBlock> fsbs = dmlp.getFunctionStatementBlocks(fnspace);
Iterator<Entry<String, FunctionStatementBlock>> iter = fsbs.entrySet().iterator();
- while( iter.hasNext() )
- {
+ while( iter.hasNext() ) {
Entry<String, FunctionStatementBlock> e = iter.next();
- String fname = e.getKey();
- String fKey = DMLProgram.constructFunctionKey(fnspace, fname);
- //probe function candidates, remove if no candidate
- if( !fcandKeys.contains(fKey) )
+ if( !fgraph.isReachableFunction(fnspace, e.getKey()) ) {
iter.remove();
+ if( LOG.isDebugEnabled() )
+ LOG.debug("IPA: Removed unused function: " +
+ DMLProgram.constructFunctionKey(fnspace, e.getKey()));
+ }
}
}
}
@@ -902,17 +899,20 @@ public class InterProceduralAnalysis
* @param dmlp the DML program
* @throws LanguageException if LanguageException occurs
*/
- public void flagFunctionsForRecompileOnce( DMLProgram dmlp )
+ public void flagFunctionsForRecompileOnce( DMLProgram dmlp, FunctionCallGraph fgraph )
throws LanguageException
{
for (String namespaceKey : dmlp.getNamespaces().keySet())
for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet())
{
FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- if( rFlagFunctionForRecompileOnce( fsblock, false ) )
+ if( !fgraph.isRecursiveFunction(namespaceKey, fname) &&
+ rFlagFunctionForRecompileOnce( fsblock, false ) )
{
fsblock.setRecompileOnce( true );
- LOG.debug("IPA: FUNC flagged for recompile-once: " + DMLProgram.constructFunctionKey(namespaceKey, fname));
+ if( LOG.isDebugEnabled() )
+ LOG.debug("IPA: FUNC flagged for recompile-once: " +
+ DMLProgram.constructFunctionKey(namespaceKey, fname));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/main/java/org/apache/sysml/parser/DMLProgram.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLProgram.java b/src/main/java/org/apache/sysml/parser/DMLProgram.java
index 292a13c..9fbd63c 100644
--- a/src/main/java/org/apache/sysml/parser/DMLProgram.java
+++ b/src/main/java/org/apache/sysml/parser/DMLProgram.java
@@ -107,6 +107,14 @@ public class DMLProgram
return namespaceProgram._functionBlocks;
}
+ public boolean hasFunctionStatementBlocks() {
+ boolean ret = false;
+ for( DMLProgram nsProg : _namespaces.values() )
+ ret |= !nsProg._functionBlocks.isEmpty();
+
+ return ret;
+ }
+
public ArrayList<FunctionStatementBlock> getFunctionStatementBlocks()
throws LanguageException
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/main/java/org/apache/sysml/utils/Explain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java
index 097e2b7..ccc9853 100644
--- a/src/main/java/org/apache/sysml/utils/Explain.java
+++ b/src/main/java/org/apache/sysml/utils/Explain.java
@@ -21,12 +21,12 @@ package org.apache.sysml.utils;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.sysml.api.DMLException;
-import org.apache.sysml.hops.FunctionOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.Hop.VisitStatus;
import org.apache.sysml.hops.HopsException;
@@ -35,6 +35,7 @@ import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.globalopt.gdfgraph.GDFLoopNode;
import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode;
import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType;
+import org.apache.sysml.hops.ipa.FunctionCallGraph;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.ForStatement;
@@ -206,32 +207,28 @@ public class Explain
sb.append("\nPROGRAM\n");
// Explain functions (if exists)
- boolean firstFunction = true;
- for (String namespace : prog.getNamespaces().keySet()) {
- for (String fname : prog.getFunctionStatementBlocks(namespace).keySet()) {
- if (firstFunction) {
- sb.append("--FUNCTIONS\n");
- firstFunction = false;
+ if( prog.hasFunctionStatementBlocks() ) {
+ sb.append("--FUNCTIONS\n");
+
+ //show function call graph
+ sb.append("----FUNCTION CALL GRAPH\n");
+ sb.append("------MAIN PROGRAM\n");
+ FunctionCallGraph fgraph = new FunctionCallGraph(prog);
+ sb.append(explainFunctionCallGraph(fgraph, new HashSet<String>(), null, 3));
+
+ //show individual functions
+ for (String namespace : prog.getNamespaces().keySet()) {
+ for (String fname : prog.getFunctionStatementBlocks(namespace).keySet()) {
+ FunctionStatementBlock fsb = prog.getFunctionStatementBlock(namespace, fname);
+ FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
- //show function call dag
- sb.append("----FUNCTION CALL DAG\n");
- sb.append("------MAIN PROGRAM\n");
- HashSet<String> fstack = new HashSet<String>();
- HashSet<String> lfset = new HashSet<String>();
- for( StatementBlock sblk : prog.getStatementBlocks() )
- sb.append(explainFunctionCallDag(sblk, fstack, lfset, 3));
- }
-
- //show individual functions
- FunctionStatementBlock fsb = prog.getFunctionStatementBlock(namespace, fname);
- FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
-
- if (fstmt instanceof ExternalFunctionStatement)
- sb.append("----EXTERNAL FUNCTION " + namespace + "::" + fname + "\n");
- else {
- sb.append("----FUNCTION " + namespace + "::" + fname + " [recompile="+fsb.isRecompileOnce()+"]\n");
- for (StatementBlock current : fstmt.getBody())
- sb.append(explainStatementBlock(current, 3));
+ if (fstmt instanceof ExternalFunctionStatement)
+ sb.append("----EXTERNAL FUNCTION " + namespace + "::" + fname + "\n");
+ else {
+ sb.append("----FUNCTION " + namespace + "::" + fname + " [recompile="+fsb.isRecompileOnce()+"]\n");
+ for (StatementBlock current : fstmt.getBody())
+ sb.append(explainStatementBlock(current, 3));
+ }
}
}
}
@@ -267,17 +264,15 @@ public class Explain
{
sb.append("--FUNCTIONS\n");
- //show function call dag
+ //show function call graph
if( !rtprog.getProgramBlocks().isEmpty() &&
rtprog.getProgramBlocks().get(0).getStatementBlock() != null )
{
- sb.append("----FUNCTION CALL DAG\n");
+ sb.append("----FUNCTION CALL GRAPH\n");
sb.append("------MAIN PROGRAM\n");
DMLProgram prog = rtprog.getProgramBlocks().get(0).getStatementBlock().getDMLProg();
- HashSet<String> fstack = new HashSet<String>();
- HashSet<String> lfset = new HashSet<String>();
- for( StatementBlock sblk : prog.getStatementBlocks() )
- sb.append(explainFunctionCallDag(sblk, fstack, lfset, 3));
+ FunctionCallGraph fgraph = new FunctionCallGraph(prog);
+ sb.append(explainFunctionCallGraph(fgraph, new HashSet<String>(), null, 3));
}
//show individual functions
@@ -932,68 +927,22 @@ public class Explain
return ret;
}
- private static String explainFunctionCallDag(StatementBlock sb, HashSet<String> fstack, HashSet<String> lfset, int level)
+ private static String explainFunctionCallGraph(FunctionCallGraph fgraph, HashSet<String> fstack, String fkey, int level)
throws HopsException
{
StringBuilder builder = new StringBuilder();
-
- if (sb instanceof WhileStatementBlock) {
- WhileStatement ws = (WhileStatement)sb.getStatement(0);
- for (StatementBlock current : ws.getBody())
- builder.append(explainFunctionCallDag(current, fstack, lfset, level));
- }
- else if (sb instanceof IfStatementBlock) {
- IfStatement ifs = (IfStatement) sb.getStatement(0);
- for (StatementBlock current : ifs.getIfBody())
- builder.append(explainFunctionCallDag(current, fstack, lfset, level));
- for (StatementBlock current : ifs.getElseBody())
- builder.append(explainFunctionCallDag(current, fstack, lfset, level));
- }
- else if (sb instanceof ForStatementBlock) {
- ForStatement fs = (ForStatement)sb.getStatement(0);
- for (StatementBlock current : fs.getBody())
- builder.append(explainFunctionCallDag(current, fstack, lfset, level));
- }
- else if (sb instanceof FunctionStatementBlock) {
- FunctionStatement fsb = (FunctionStatement) sb.getStatement(0);
- for (StatementBlock current : fsb.getBody())
- builder.append(explainFunctionCallDag(current, fstack, lfset, level));
- }
- else {
- // For generic StatementBlock
- ArrayList<Hop> hopsDAG = sb.get_hops();
- if( hopsDAG != null && !hopsDAG.isEmpty() ) {
- //function ops can only occur as root nodes of the dag
- for( Hop h : hopsDAG )
- if( h instanceof FunctionOp ){
- FunctionOp fop = (FunctionOp) h;
- String fkey = DMLProgram.constructFunctionKey(fop.getFunctionNamespace(), fop.getFunctionName());
- //prevent redundant call edges
- if( !lfset.contains(fkey) && !fop.getFunctionNamespace().equals(DMLProgram.INTERNAL_NAMESPACE) )
- {
- //recursively explain function call dag
- if( !fstack.contains(fkey) ) {
- fstack.add(fkey);
- String offset = createOffset(level);
- builder.append(offset + "--" + fkey + "\n");
- FunctionStatementBlock fsb = sb.getDMLProg()
- .getFunctionStatementBlock(fop.getFunctionNamespace(), fop.getFunctionName());
- FunctionStatement fs = (FunctionStatement) fsb.getStatement(0);
- HashSet<String> lfset2 = new HashSet<String>();
- for( StatementBlock csb : fs.getBody() )
- builder.append(explainFunctionCallDag(csb, fstack, lfset2, level+1));
- fstack.remove(fkey);
- }
- //recursive function call
- else {
- String offset = createOffset(level);
- builder.append(offset + "-->" + fkey + " (recursive)\n");
- }
-
- //mark as visited for current function call context
- lfset.add( fkey );
- }
- }
+ String offset = createOffset(level);
+ Collection<String> cfkeys = fgraph.getCalledFunctions(fkey);
+ if( cfkeys != null ) {
+ for( String cfkey : cfkeys ) {
+ if( fstack.contains(cfkey) && fgraph.isRecursiveFunction(cfkey) )
+ builder.append(offset + "--" + cfkey + " (recursive)\n");
+ else {
+ fstack.add(cfkey);
+ builder.append(offset + "--" + cfkey + "\n");
+ builder.append(explainFunctionCallGraph(fgraph, fstack, cfkey, level+1));
+ fstack.remove(cfkey);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/main/java/org/apache/sysml/utils/Statistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java
index 87be90b..cf9b5fb 100644
--- a/src/main/java/org/apache/sysml/utils/Statistics.java
+++ b/src/main/java/org/apache/sysml/utils/Statistics.java
@@ -345,6 +345,9 @@ public class Statistics
hopRecompilePred.set(0);
hopRecompileSB.set(0);
+ funRecompiles.set(0);
+ funRecompileTime.set(0);
+
parforOptCount = 0;
parforOptTime = 0;
parforInitTime = 0;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test/java/org/apache/sysml/test/integration/functions/recompile/RecursiveFunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RecursiveFunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RecursiveFunctionRecompileTest.java
new file mode 100644
index 0000000..99459b0
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RecursiveFunctionRecompileTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.recompile;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+
+/**
+ * This test ensures that recursive functions are not marked for recompile-once
+ * during IPA because this could potentially lead to incorrect plans that cause
+ * OOMs or even incorrect results.
+ *
+ */
+public class RecursiveFunctionRecompileTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/recompile/";
+ private final static String TEST_NAME1 = "recursive_func_direct";
+ private final static String TEST_NAME2 = "recursive_func_indirect";
+ private final static String TEST_NAME3 = "recursive_func_indirect2";
+ private final static String TEST_NAME4 = "recursive_func_none";
+ private final static String TEST_CLASS_DIR = TEST_DIR +
+ RecursiveFunctionRecompileTest.class.getSimpleName() + "/";
+
+ private final static long rows = 5000;
+ private final static long cols = 10000;
+ private final static double sparsity = 0.00001d;
+ private final static double val = 7.0;
+
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "Rout" }) );
+ addTestConfiguration(TEST_NAME2,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "Rout" }) );
+ addTestConfiguration(TEST_NAME3,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "Rout" }) );
+ addTestConfiguration(TEST_NAME4,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "Rout" }) );
+ }
+
+ @Test
+ public void testDirectRecursionRecompileIPA() {
+ runRecompileTest(TEST_NAME1, true);
+ }
+
+ @Test
+ public void testIndirectRecursionRecompileIPA() {
+ runRecompileTest(TEST_NAME2, true);
+ }
+
+ @Test
+ public void testIndirect2RecursionRecompileIPA() {
+ runRecompileTest(TEST_NAME3, true);
+ }
+
+ @Test
+ public void testNoRecursionRecompileIPA() {
+ runRecompileTest(TEST_NAME4, true);
+ }
+
+ @Test
+ public void testDirectRecursionRecompileNoIPA() {
+ runRecompileTest(TEST_NAME1, false);
+ }
+
+ @Test
+ public void testIndirectRecursionRecompileNoIPA() {
+ runRecompileTest(TEST_NAME2, false);
+ }
+
+ @Test
+ public void testIndirect2RecursionRecompileNoIPA() {
+ runRecompileTest(TEST_NAME3, false);
+ }
+
+ @Test
+ public void testNoRecursionRecompileNoIPA() {
+ runRecompileTest(TEST_NAME4, false);
+ }
+
+ private void runRecompileTest( String testname, boolean IPA )
+ {
+ boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
+
+ try
+ {
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain","-stats","-args",
+ input("V"), Double.toString(val), output("R") };
+
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
+
+ //generate sparse input data
+ MatrixBlock mb = MatrixBlock.randOperations((int)rows, (int)cols, sparsity, 0, 1, "uniform", 732);
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows,cols,OptimizerUtils.DEFAULT_BLOCKSIZE,OptimizerUtils.DEFAULT_BLOCKSIZE,(long)(rows*cols*sparsity));
+ DataConverter.writeMatrixToHDFS(mb, input("V"), OutputInfo.TextCellOutputInfo, mc);
+ MapReduceTool.writeMetaDataFile(input("V.mtd"), ValueType.DOUBLE, mc, OutputInfo.TextCellOutputInfo);
+
+ //run test
+ runTest(true, false, null, -1);
+
+ //check number of recompiled functions (recompile_once is not applicable for recursive functions
+ //because the single recompilation on entry would implicitly change the remaining plan of the caller;
+ //if not not handled correctly, TEST_NAME1 and TEST_NAME2 would have show with IPA 1111 function recompilations.
+ Assert.assertEquals(testname.equals(TEST_NAME4) && IPA ? 1 : 0, Statistics.getFunRecompiles());
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ Assert.fail("Failed to run test: "+ex.getMessage());
+ }
+ finally {
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test/scripts/functions/recompile/recursive_func_direct.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/recursive_func_direct.dml b/src/test/scripts/functions/recompile/recursive_func_direct.dml
new file mode 100644
index 0000000..368bbc7
--- /dev/null
+++ b/src/test/scripts/functions/recompile/recursive_func_direct.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo1 = function (Matrix[Double] X)
+ return (Matrix[Double] Y)
+{
+ print(ncol(X)+" cols -> "+sum(X));
+ for( i in 1:10 ) {
+ batch = ncol(X)/10;
+ tmp = X[,((i-1)*batch+1):(i*batch)];
+ if( batch > 1 )
+ tmp = foo1(tmp);
+ }
+ Y = X*2;
+}
+
+V = read($1);
+V = foo1(V);
+print(sum(V));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test/scripts/functions/recompile/recursive_func_indirect.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/recursive_func_indirect.dml b/src/test/scripts/functions/recompile/recursive_func_indirect.dml
new file mode 100644
index 0000000..0e7b023
--- /dev/null
+++ b/src/test/scripts/functions/recompile/recursive_func_indirect.dml
@@ -0,0 +1,46 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo1 = function (Matrix[Double] X)
+ return (Matrix[Double] Y)
+{
+ print(ncol(X)+" cols -> "+sum(X));
+ for( i in 1:10 ) {
+ batch = ncol(X)/10;
+ tmp = X[,((i-1)*batch+1):(i*batch)];
+ tmp = foo2(tmp, batch);
+ }
+ Y = X*2;
+}
+
+foo2 = function (Matrix[Double] X, Integer batch)
+ return (Matrix[Double] Y)
+{
+ if( batch > 1 )
+ Y = foo1(X);
+ else
+ Y = X;
+}
+
+V = read($1);
+V = foo1(V);
+print(sum(V));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test/scripts/functions/recompile/recursive_func_indirect2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/recursive_func_indirect2.dml b/src/test/scripts/functions/recompile/recursive_func_indirect2.dml
new file mode 100644
index 0000000..f30f580
--- /dev/null
+++ b/src/test/scripts/functions/recompile/recursive_func_indirect2.dml
@@ -0,0 +1,55 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo1 = function (Matrix[Double] X)
+ return (Matrix[Double] Y)
+{
+ print(ncol(X)+" cols -> "+sum(X));
+ for( i in 1:10 ) {
+ batch = ncol(X)/10;
+ tmp = X[,((i-1)*batch+1):(i*batch)];
+ tmp = foo2(tmp, batch);
+ }
+ Y = X*2;
+}
+
+foo2 = function (Matrix[Double] X, Integer batch)
+ return (Matrix[Double] Y)
+{
+ if( batch > 1 )
+ Y = foo3(X);
+ else
+ Y = X;
+}
+
+foo3 = function (Matrix[Double] X)
+ return (Matrix[Double] Y)
+{
+ if( sum(X) >= 0 )
+ Y = foo1(X);
+ else
+ Y = X;
+}
+
+V = read($1);
+V = foo3(V);
+print(sum(V));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test/scripts/functions/recompile/recursive_func_none.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/recompile/recursive_func_none.dml b/src/test/scripts/functions/recompile/recursive_func_none.dml
new file mode 100644
index 0000000..7548017
--- /dev/null
+++ b/src/test/scripts/functions/recompile/recursive_func_none.dml
@@ -0,0 +1,39 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo1 = function (Matrix[Double] X)
+ return (Matrix[Double] Y)
+{
+ print(ncol(X)+" cols -> "+sum(X));
+ for( i in 1:10 ) {
+ batch = ncol(X)/10;
+ tmp = X[,((i-1)*batch+1):(i*batch)];
+ if( sum(tmp)<0 )
+ print("Sum is negative");
+ }
+ Y = X*2;
+}
+
+V = read($1);
+V = foo1(V);
+print(sum(V));
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0a3d2481/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
index c15340a..da37e50 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/recompile/ZPackageSuite.java
@@ -40,6 +40,7 @@ import org.junit.runners.Suite;
RandRecompileTest.class,
RandSizeExpressionEvalTest.class,
ReblockRecompileTest.class,
+ RecursiveFunctionRecompileTest.class,
RemoveEmptyPotpourriTest.class,
RemoveEmptyRecompileTest.class,
RewriteComplexMapMultChainTest.class,