You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/03/12 07:07:56 UTC
incubator-systemml git commit: [SYSTEMML-1395] Robustness of parfor
spark partitioning w/ unknown nnz
Repository: incubator-systemml
Updated Branches:
refs/heads/master 28fe4fe8f -> bcf96e1cc
[SYSTEMML-1395] Robustness of parfor spark partitioning w/ unknown nnz
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/bcf96e1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/bcf96e1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/bcf96e1c
Branch: refs/heads/master
Commit: bcf96e1ccc32948fa2420bd5d9649d15a8d4d521
Parents: 28fe4fe
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Mar 11 23:08:48 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Mar 11 23:08:48 2017 -0800
----------------------------------------------------------------------
.../sysml/runtime/controlprogram/ParForProgramBlock.java | 10 ++++++----
.../runtime/controlprogram/caching/MatrixObject.java | 2 +-
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 3d1e6ee..3339dca 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1068,11 +1068,13 @@ public class ParForProgramBlock extends ForProgramBlock
//write matrices to HDFS, except DP matrix which is the input to the RemoteDPParForSpark job
exportMatricesToHDFS(ec, _colocatedDPMatrix);
-
+
// Step 4) submit MR job (wait for finished work)
- OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
- (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))?
- OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
+ //TODO runtime support for binary cell partitioning
+ //OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
+ // (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))?
+ // OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
+ OutputInfo inputOI = OutputInfo.BinaryBlockOutputInfo;
RemoteParForJobReturn ret = RemoteDPParForSpark.runJob(_ID, itervar.getName(), _colocatedDPMatrix, program, clsMap,
resultFile, inputMatrix, ec, inputDPF, inputOI, _tSparseCol, _enableCPCaching, _numThreads );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index b73adb0..48855ee 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -185,7 +185,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
public double getSparsity() {
MatrixCharacteristics mc = getMatrixCharacteristics();
- return ((double)mc.getNonZeros())/mc.getRows()/mc.getCols();
+ return OptimizerUtils.getSparsity(mc);
}
// *********************************************