You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/11/19 21:46:49 UTC
[07/50] [abbrv] incubator-systemml git commit: Fix mr textcell
reblock number of reducers (robustness unknown nnz)
Fix mr textcell reblock number of reducers (robustness unknown nnz)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/142b9f63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/142b9f63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/142b9f63
Branch: refs/heads/master
Commit: 142b9f63bb288d30f10127119a81fd7e99beab88
Parents: 28bc17e
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Oct 27 13:29:48 2015 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Oct 28 10:33:39 2015 -0700
----------------------------------------------------------------------
src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/142b9f63/src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java b/src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java
index 573765b..f125425 100644
--- a/src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java
+++ b/src/main/java/com/ibm/bi/dml/runtime/matrix/ReblockMR.java
@@ -204,9 +204,9 @@ public class ReblockMR
long maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
long blockSize = InfrastructureAnalyzer.getHDFSBlockSize()/(1024*1024);
long maxSize = -1; //in MB
- for( int i=0; i<rlen.length; i++ )
- {
- long tmp = MatrixBlock.estimateSizeOnDisk(rlen[i], clen[i], nnz[i]) / (1024*1024);
+ for( int i=0; i<rlen.length; i++ ) {
+ long lnnz = (nnz[i] > 0) ? nnz[i] : rlen[i] * clen[i];
+ long tmp = MatrixBlock.estimateSizeOnDisk(rlen[i], clen[i], lnnz) / (1024*1024);
maxSize = Math.max(maxSize, tmp);
}