You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/03/12 07:07:56 UTC

incubator-systemml git commit: [SYSTEMML-1395] Robustness of parfor spark partitioning w/ unknown nnz

Repository: incubator-systemml
Updated Branches:
  refs/heads/master 28fe4fe8f -> bcf96e1cc


[SYSTEMML-1395] Robustness of parfor spark partitioning w/ unknown nnz

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/bcf96e1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/bcf96e1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/bcf96e1c

Branch: refs/heads/master
Commit: bcf96e1ccc32948fa2420bd5d9649d15a8d4d521
Parents: 28fe4fe
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Mar 11 23:08:48 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Mar 11 23:08:48 2017 -0800

----------------------------------------------------------------------
 .../sysml/runtime/controlprogram/ParForProgramBlock.java  | 10 ++++++----
 .../runtime/controlprogram/caching/MatrixObject.java      |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 3d1e6ee..3339dca 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1068,11 +1068,13 @@ public class ParForProgramBlock extends ForProgramBlock
 		
 		//write matrices to HDFS, except DP matrix which is the input to the RemoteDPParForSpark job
 		exportMatricesToHDFS(ec, _colocatedDPMatrix);
-				
+		
 		// Step 4) submit MR job (wait for finished work)
-		OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
-				              (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))? 
-				             OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
+		//TODO runtime support for binary cell partitioning 
+		//OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)||
+		//		              (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))? 
+		//		             OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo;
+		OutputInfo inputOI = OutputInfo.BinaryBlockOutputInfo;
 		RemoteParForJobReturn ret = RemoteDPParForSpark.runJob(_ID, itervar.getName(), _colocatedDPMatrix, program, clsMap, 
 				resultFile, inputMatrix, ec, inputDPF, inputOI, _tSparseCol, _enableCPCaching, _numThreads );
 		

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index b73adb0..48855ee 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -185,7 +185,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
 
 	public double getSparsity() {
 		MatrixCharacteristics mc = getMatrixCharacteristics();		
-		return ((double)mc.getNonZeros())/mc.getRows()/mc.getCols();
+		return OptimizerUtils.getSparsity(mc);
 	}
 	
 	// *********************************************