You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/03/10 06:25:18 UTC
[1/5] incubator-systemml git commit: [SYSTEMML-1381] Fix worst-case
size propagation convolution hop
Repository: incubator-systemml
Updated Branches:
refs/heads/master 0a61fe084 -> e3a75d141
[SYSTEMML-1381] Fix worst-case size propagation convolution hop
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/442b9a5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/442b9a5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/442b9a5b
Branch: refs/heads/master
Commit: 442b9a5b4aaca57f0d459da267551c885a35e6e1
Parents: 0a61fe0
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Mar 9 17:25:51 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Mar 9 17:25:51 2017 -0800
----------------------------------------------------------------------
.../org/apache/sysml/hops/ConvolutionOp.java | 33 ++++++--------------
1 file changed, 9 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/442b9a5b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
index a13de52..c32f227 100644
--- a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
+++ b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java
@@ -244,15 +244,14 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
protected long[] inferOutputCharacteristics( MemoTable memo )
{
// [numRows, numCols, NNZ]
- long[] ret = null;
+ long[] ret = new long[3];
if(op == ConvOp.BIAS_ADD) {
MatrixCharacteristics[] mc = memo.getAllInputStats(getInput());
- ret = new long[3];
ret[0] = mc[0].rowsKnown() ? mc[0].getRows() : -1;
ret[1] = mc[0].colsKnown() ? mc[0].getCols() : -1;
ret[2] = -1;
- return ret;
+ return (ret[0]>0 && ret[1]>0) ? ret : null;
}
ConvolutionParameters params;
@@ -264,41 +263,26 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
switch(op)
{
- case MAX_POOLING:
- {
- ret = new long[3];
+ case MAX_POOLING: {
ret[0] = getInput().get(0)._dim1;
ret[1] = getExtractedVal(params.C, params.P, params.Q);
ret[2] = -1;
break;
}
- case MAX_POOLING_BACKWARD:
- {
- ret = new long[3];
- ret[0] = getInput().get(0)._dim1;
- ret[1] = getInput().get(0)._dim2;
- ret[2] = -1;
- break;
- }
- case DIRECT_CONV2D:
- {
- ret = new long[3];
+ case DIRECT_CONV2D: {
ret[0] = getInput().get(0)._dim1;
ret[1] = getExtractedVal(getInput().get(1)._dim1, params.P, params.Q);
ret[2] = -1;
break;
}
- case DIRECT_CONV2D_BACKWARD_FILTER:
- {
- ret = new long[3];
+ case DIRECT_CONV2D_BACKWARD_FILTER: {
ret[0] = getInput().get(1)._dim1;
ret[1] = getInput().get(1)._dim2;
ret[2] = -1;
break;
}
- case DIRECT_CONV2D_BACKWARD_DATA:
- {
- ret = new long[3];
+ case MAX_POOLING_BACKWARD:
+ case DIRECT_CONV2D_BACKWARD_DATA: {
ret[0] = getInput().get(0)._dim1;
ret[1] = getInput().get(0)._dim2;
ret[2] = -1;
@@ -316,7 +300,8 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop
" pad=[" + params.pad_h + " " + params.pad_w + "]");
}
- return ret;
+ //safe return (create entry only if at least dims known)
+ return (ret[0]>0 && ret[1]>0) ? ret : null;
}
[3/5] incubator-systemml git commit: [SYSTEMML-1386] Fix execution
context creation w/ forced singlenode
Posted by mb...@apache.org.
[SYSTEMML-1386] Fix execution context creation w/ forced singlenode
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b945543d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b945543d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b945543d
Branch: refs/heads/master
Commit: b945543d933c37ad92adc04ccbfa51410434e777
Parents: feb1cb7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Mar 9 17:58:44 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Mar 9 17:58:44 2017 -0800
----------------------------------------------------------------------
.../controlprogram/context/ExecutionContextFactory.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b945543d/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
index 4b2d9d0..9fc871b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContextFactory.java
@@ -20,6 +20,8 @@
package org.apache.sysml.runtime.controlprogram.context;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.Program;
public class ExecutionContextFactory
@@ -41,6 +43,14 @@ public class ExecutionContextFactory
switch( DMLScript.rtplatform )
{
case SINGLE_NODE:
+ //NOTE: even in case of forced singlenode operations, users might still
+ //want to run remote parfor which requires the correct execution context
+ if( OptimizerUtils.getDefaultExecutionMode()==RUNTIME_PLATFORM.HYBRID)
+ ec = new ExecutionContext(allocateVars, prog);
+ else
+ ec = new SparkExecutionContext(allocateVars, prog);
+ break;
+
case HADOOP:
case HYBRID:
ec = new ExecutionContext(allocateVars, prog);
[5/5] incubator-systemml git commit: [SYSTEMML-1387] Fix parfor
constrained optimizer (fused dpe selection)
Posted by mb...@apache.org.
[SYSTEMML-1387] Fix parfor constrained optimizer (fused dpe selection)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/e3a75d14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/e3a75d14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/e3a75d14
Branch: refs/heads/master
Commit: e3a75d1412c1b3d159e05cc877dd6e90fbdf5153
Parents: 600e641
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Mar 9 18:56:40 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Mar 9 18:56:40 2017 -0800
----------------------------------------------------------------------
.../runtime/controlprogram/parfor/opt/OptimizerRuleBased.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e3a75d14/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index a84f93f..90db845 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -1499,8 +1499,8 @@ public class OptimizerRuleBased extends Optimizer
// (this also implies that the body is CP only)
// try to merge MR data partitioning and MR exec
- if( (pn.getExecType()==ExecType.MR || pn.getExecType()==ExecType.SPARK) //MR/SP EXEC and CP body
- && M < _rm2 //fits into remote memory of reducers
+ if( (pn.getExecType()==ExecType.MR && M < _rm2 //fits into remote memory of reducers
+ || pn.getExecType()==ExecType.SPARK) //MR/SP EXEC and CP body
&& partitioner!=null && partitioner.equals(REMOTE_DP.toString()) //MR/SP partitioning
&& partitionedMatrices.size()==1 ) //only one partitioned matrix
{
[2/5] incubator-systemml git commit: [MINOR] Cleanups of mlcontext
testsuite and gpu javadoc warnings
Posted by mb...@apache.org.
[MINOR] Cleanups of mlcontext testsuite and gpu javadoc warnings
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/feb1cb72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/feb1cb72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/feb1cb72
Branch: refs/heads/master
Commit: feb1cb72dfc742ff7f181f1025d217d9efdf51b5
Parents: 442b9a5
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Mar 9 17:45:56 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Mar 9 17:45:56 2017 -0800
----------------------------------------------------------------------
.../sysml/runtime/instructions/gpu/context/GPUObject.java | 4 ++--
.../runtime/instructions/gpu/context/JCudaObject.java | 10 +++++-----
.../sysml/test/integration/mlcontext/ZPackageSuite.java | 9 +++++----
3 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/feb1cb72/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java
index de1459e..366713f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java
@@ -58,7 +58,7 @@ public abstract class GPUObject
/**
* Signal intent that a matrix block will be read (as input) on the GPU
* @return true if a host memory to device memory transfer happened
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
public abstract boolean acquireDeviceRead() throws DMLRuntimeException;
/**
@@ -84,7 +84,7 @@ public abstract class GPUObject
/**
* Signal intent that a block needs to be read on the host
* @return true if copied from device to host
- * @throws CacheException
+ * @throws CacheException ?
*/
public abstract boolean acquireHostRead() throws CacheException;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/feb1cb72/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaObject.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaObject.java
index 74b5050..c256be3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaObject.java
@@ -242,7 +242,7 @@ public class JCudaObject extends GPUObject {
* @param handle a valid {@link cusparseHandle}
* @param C Output matrix
* @param rowsC number of rows in C
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
private static void step1AllocateRowPointers(cusparseHandle handle, CSRPointer C, int rowsC) throws DMLRuntimeException {
cusparseSetPointerMode(handle, cusparsePointerMode.CUSPARSE_POINTER_MODE_HOST);
@@ -261,7 +261,7 @@ public class JCudaObject extends GPUObject {
* @param C Output Sparse Matrix C on GPU
* @param m Rows in C
* @param n Columns in C
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
private static void step2GatherNNZGeam(cusparseHandle handle, CSRPointer A, CSRPointer B, CSRPointer C, int m, int n) throws DMLRuntimeException {
int[] CnnzArray = { -1 };
@@ -293,7 +293,7 @@ public class JCudaObject extends GPUObject {
* @param m Number of rows of sparse matrix op ( A ) and C
* @param n Number of columns of sparse matrix op ( B ) and C
* @param k Number of columns/rows of sparse matrix op ( A ) / op ( B )
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
private static void step2GatherNNZGemm(cusparseHandle handle, CSRPointer A, int transA, CSRPointer B, int transB, CSRPointer C, int m, int n, int k) throws DMLRuntimeException {
int[] CnnzArray = { -1 };
@@ -321,7 +321,7 @@ public class JCudaObject extends GPUObject {
*
* @param handle a valid {@link cusparseHandle}
* @param C Output sparse matrix on GPU
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
private static void step3AllocateValNInd(cusparseHandle handle, CSRPointer C) throws DMLRuntimeException {
// Increment cudaCount by one when all three arrays of CSR sparse array are allocated
@@ -1074,7 +1074,7 @@ public class JCudaObject extends GPUObject {
* Convert sparse to dense (Performs transpose, use sparseToColumnMajorDense if the kernel can deal with column major format)
* Also records per instruction invokation of sparseToDense.
* @param instructionName Name of the instruction for which statistics are recorded in {@link GPUStatistics}
- * @throws DMLRuntimeException
+ * @throws DMLRuntimeException ?
*/
public void sparseToDense(String instructionName) throws DMLRuntimeException {
long start = System.nanoTime();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/feb1cb72/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
index 9bb1a55..44f3533 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
@@ -27,10 +27,11 @@ import org.junit.runners.Suite;
* they should not be run in parallel. */
@RunWith(Suite.class)
@Suite.SuiteClasses({
- org.apache.sysml.test.integration.mlcontext.MLContextFrameTest.class,
- org.apache.sysml.test.integration.mlcontext.MLContextMultipleScriptsTest.class,
- org.apache.sysml.test.integration.mlcontext.MLContextScratchCleanupTest.class,
- org.apache.sysml.test.integration.mlcontext.MLContextTest.class
+ MLContextFrameTest.class,
+ MLContextMultipleScriptsTest.class,
+ MLContextParforDatasetTest.class,
+ MLContextScratchCleanupTest.class,
+ MLContextTest.class
})
[4/5] incubator-systemml git commit: [SYSTEMML-1388] Fix load
imbalance parfor spark w/ factoring
Posted by mb...@apache.org.
[SYSTEMML-1388] Fix load imbalance parfor spark w/ factoring
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/600e641b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/600e641b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/600e641b
Branch: refs/heads/master
Commit: 600e641b2c5e39fca1e7cde197f1f1e26ce112b6
Parents: b945543
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Mar 9 18:51:21 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Mar 9 18:51:21 2017 -0800
----------------------------------------------------------------------
.../runtime/controlprogram/parfor/RemoteParForSpark.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/600e641b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
index 5e69b19..49ac9db 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java
@@ -71,10 +71,10 @@ public class RemoteParForSpark
//run remote_spark parfor job
//(w/o lazy evaluation to fit existing parfor framework, e.g., result merge)
RemoteParForSparkWorker func = new RemoteParForSparkWorker(program, clsMap, cpCaching, aTasks, aIters);
- List<Tuple2<Long,String>> out =
- sc.parallelize( tasks, numMappers ) //create rdd of parfor tasks
- .flatMapToPair( func ) //execute parfor tasks
- .collect(); //get output handles
+ List<Tuple2<Long,String>> out = sc
+ .parallelize(tasks, tasks.size()) //create rdd of parfor tasks
+ .flatMapToPair(func) //execute parfor tasks
+ .collect(); //get output handles
//de-serialize results
LocalVariableMap[] results = RemoteParForUtils.getResults(out, LOG);