You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/05 02:55:16 UTC
systemml git commit: [HOTFIX][SYSTEMML-2297] Fix spark left indexing
ops over frames
Repository: systemml
Updated Branches:
refs/heads/master 1791c1a26 -> 6d2140224
[HOTFIX][SYSTEMML-2297] Fix spark left indexing ops over frames
The recent fix for matrix left indexing with partitioned broadcasts
revealed a hidden issue with spark left indexing operations over frames,
which under certain conditions, did not use the correct block sizes.
This patch fixes the root cause and cleans up the related slice
operations over broadcasts to avoid unnecessary reflection.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/6d214022
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/6d214022
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/6d214022
Branch: refs/heads/master
Commit: 6d2140224e22a524a13f080ff8c4a49a69ab6284
Parents: 1791c1a
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri May 4 19:56:14 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri May 4 19:56:14 2018 -0700
----------------------------------------------------------------------
.../context/SparkExecutionContext.java | 9 +++----
.../spark/data/PartitionedBroadcast.java | 25 ++++----------------
.../runtime/matrix/MatrixCharacteristics.java | 7 +++---
3 files changed, 14 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/6d214022/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 8e7d888..4196ef8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -654,11 +654,12 @@ public class SparkExecutionContext extends ExecutionContext
if (!isLocalMaster())
pmb.clearBlocks();
}
-
- bret = new PartitionedBroadcast<>(ret, fo.getMatrixCharacteristics());
- if (fo.getBroadcastHandle() == null) {
+
+ bret = new PartitionedBroadcast<>(ret, new MatrixCharacteristics(
+ fo.getMatrixCharacteristics()).setBlockSize(brlen, bclen));
+ if (fo.getBroadcastHandle() == null)
fo.setBroadcastHandle(new BroadcastObject<FrameBlock>());
- }
+
fo.getBroadcastHandle().setPartitionedBroadcast(bret,
OptimizerUtils.estimatePartitionedSizeExactSparsity(fo.getMatrixCharacteristics()));
CacheableData.addBroadcastSize(fo.getBroadcastHandle().getPartitionedBroadcastSize());
http://git-wip-us.apache.org/repos/asf/systemml/blob/6d214022/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
index a4c5173..ab29c4c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java
@@ -20,11 +20,9 @@
package org.apache.sysml.runtime.instructions.spark.data;
import java.io.Serializable;
-import java.lang.reflect.Constructor;
import java.util.ArrayList;
import org.apache.spark.broadcast.Broadcast;
-import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysml.runtime.controlprogram.caching.CacheBlockFactory;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
@@ -121,7 +119,7 @@ public class PartitionedBroadcast<T extends CacheBlock> implements Serializable
int end_iix = (lru-1)/_mc.getRowsPerBlock()+1;
int start_jix = (lcl-1)/_mc.getColsPerBlock()+1;
int end_jix = (lcu-1)/_mc.getColsPerBlock()+1;
-
+
for( int iix = start_iix; iix <= end_iix; iix++ )
for(int jix = start_jix; jix <= end_jix; jix++) {
IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
@@ -129,23 +127,10 @@ public class PartitionedBroadcast<T extends CacheBlock> implements Serializable
ixrange, _mc.getRowsPerBlock(), _mc.getColsPerBlock(), iix, jix, getBlock(iix, jix)));
}
- if(allBlks.size() == 1) {
- return (T) allBlks.get(0).getValue();
- }
- else {
- //allocate output matrix
- Constructor<?> constr;
- try {
- constr = block.getClass().getConstructor(int.class, int.class, boolean.class);
- T ret = (T) constr.newInstance(lru-lrl+1, lcu-lcl+1, false);
- for(Pair<?, ?> kv : allBlks) {
- ret.merge((T)kv.getValue(), false);
- }
- return ret;
- } catch (Exception e) {
- throw new DMLRuntimeException(e);
- }
- }
+ T ret = (T) allBlks.get(0).getValue();
+ for(int i=1; i<allBlks.size(); i++)
+ ret.merge((T)allBlks.get(i).getValue(), false);
+ return ret;
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/6d214022/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
index 91d70f8..6d816b6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
@@ -179,13 +179,14 @@ public class MatrixCharacteristics implements Serializable
numColumns = nc;
}
- public void setBlockSize(int blen) {
- setBlockSize(blen, blen);
+ public MatrixCharacteristics setBlockSize(int blen) {
+ return setBlockSize(blen, blen);
}
- public void setBlockSize(int bnr, int bnc) {
+ public MatrixCharacteristics setBlockSize(int bnr, int bnc) {
numRowsPerBlock = bnr;
numColumnsPerBlock = bnc;
+ return this;
}
public void setNonZeros(long nnz) {