You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/03/14 07:11:11 UTC
incubator-systemml git commit: [SYSTEMML-1367] Performance
parfor-dpesp row input handling
Repository: incubator-systemml
Updated Branches:
refs/heads/master e81b9059e -> 19e46bb17
[SYSTEMML-1367] Performance parfor-dpesp row input handling
Recently we added the ability to pass entire rows to parfor
datapartition-execute without the need for grouping. This applies to
rows smaller than the blocksize or input datasets with arbitrary large
rows. This patch improves performance for these cases, by avoiding
unnecessary copies of single fragment partitions to the target working
partition. Furthermore, it reduces memory pressure as it reduces the
working set size by #cores * partition size.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/19e46bb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/19e46bb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/19e46bb1
Branch: refs/heads/master
Commit: 19e46bb171cf4b2abe05f841040de2f136b97b4f
Parents: e81b905
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Mar 14 00:10:18 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Mar 14 00:11:25 2017 -0700
----------------------------------------------------------------------
.../controlprogram/parfor/RemoteDPParForSparkWorker.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/19e46bb1/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
index 637d11f..400fbd5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java
@@ -21,6 +21,7 @@ package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;
@@ -198,6 +199,14 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF
private MatrixBlock collectBinaryBlock( Iterable<Writable> valueList, MatrixBlock reuse )
throws IOException
{
+ //fast path for partition of single fragment (see pseudo grouping),
+ //which avoids unnecessary copies and reduces memory pressure
+ if( valueList instanceof Collection && ((Collection<Writable>)valueList).size()==1 ) {
+ return ((PairWritableBlock)valueList.iterator().next()).block;
+ }
+
+ //default: create or reuse target partition and copy individual partition fragments
+ //into this target, including nnz maintenance and potential dense-sparse format change
MatrixBlock partition = reuse;
try